projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: allow semaphore emission to fail
[linux-block.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
8cc1a532
AD		 30#include "cikd.h"
31#include "atom.h"
841cf442 32#include "cik_blit_shaders.h"
8c68e393 33#include "radeon_ucode.h"
22c775ce 34#include "clearstate_ci.h"
02c81327
AD		 35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
d4775655
AD		 44MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
02c81327
AD		 52MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 57MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
02c81327
AD		 58MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59MODULE_FIRMWARE("radeon/KABINI_me.bin");
60MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 63MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
02c81327 64
a59781bb
AD		 65extern int r600_ih_ring_alloc(struct radeon_device *rdev);
66extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 67extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 69extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 70extern void sumo_rlc_fini(struct radeon_device *rdev);
71extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 72extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 73extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 74extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
2483b4ea
CK		 75extern int cik_sdma_resume(struct radeon_device *rdev);
76extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
77extern void cik_sdma_fini(struct radeon_device *rdev);
cc066715 78static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 79static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 80static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 81static void cik_init_pg(struct radeon_device *rdev);
82static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 83static void cik_fini_pg(struct radeon_device *rdev);
84static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 85static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
86 bool enable);
6f2043ce 87
286d9cc6
AD		 88/* get temperature in millidegrees */
89int ci_get_temp(struct radeon_device *rdev)
90{
91 u32 temp;
92 int actual_temp = 0;
93
94 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
95 CTF_TEMP_SHIFT;
96
97 if (temp & 0x200)
98 actual_temp = 255;
99 else
100 actual_temp = temp & 0x1ff;
101
102 actual_temp = actual_temp * 1000;
103
104 return actual_temp;
105}
106
107/* get temperature in millidegrees */
108int kv_get_temp(struct radeon_device *rdev)
109{
110 u32 temp;
111 int actual_temp = 0;
112
113 temp = RREG32_SMC(0xC0300E0C);
114
115 if (temp)
116 actual_temp = (temp / 8) - 49;
117 else
118 actual_temp = 0;
119
120 actual_temp = actual_temp * 1000;
121
122 return actual_temp;
123}
6f2043ce 124
6e2c3c0a
AD		 125/*
126 * Indirect registers accessor
127 */
128u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
129{
0a5b7b0b 130 unsigned long flags;
6e2c3c0a
AD		 131 u32 r;
132
0a5b7b0b 133 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 134 WREG32(PCIE_INDEX, reg);
135 (void)RREG32(PCIE_INDEX);
136 r = RREG32(PCIE_DATA);
0a5b7b0b 137 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 138 return r;
139}
140
141void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
142{
0a5b7b0b
AD		 143 unsigned long flags;
144
145 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 146 WREG32(PCIE_INDEX, reg);
147 (void)RREG32(PCIE_INDEX);
148 WREG32(PCIE_DATA, v);
149 (void)RREG32(PCIE_DATA);
0a5b7b0b 150 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 151}
152
22c775ce
AD		 153static const u32 spectre_rlc_save_restore_register_list[] =
154{
155 (0x0e00 << 16) | (0xc12c >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc140 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc150 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc15c >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc168 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0xc170 >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc178 >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0xc204 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0xc2b4 >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0xc2b8 >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0xc2bc >> 2),
176 0x00000000,
177 (0x0e00 << 16) | (0xc2c0 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x8228 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0x829c >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0x869c >> 2),
184 0x00000000,
185 (0x0600 << 16) | (0x98f4 >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x98f8 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x9900 >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0xc260 >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0x90e8 >> 2),
194 0x00000000,
195 (0x0e00 << 16) | (0x3c000 >> 2),
196 0x00000000,
197 (0x0e00 << 16) | (0x3c00c >> 2),
198 0x00000000,
199 (0x0e00 << 16) | (0x8c1c >> 2),
200 0x00000000,
201 (0x0e00 << 16) | (0x9700 >> 2),
202 0x00000000,
203 (0x0e00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0x4e00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0x5e00 << 16) | (0xcd20 >> 2),
208 0x00000000,
209 (0x6e00 << 16) | (0xcd20 >> 2),
210 0x00000000,
211 (0x7e00 << 16) | (0xcd20 >> 2),
212 0x00000000,
213 (0x8e00 << 16) | (0xcd20 >> 2),
214 0x00000000,
215 (0x9e00 << 16) | (0xcd20 >> 2),
216 0x00000000,
217 (0xae00 << 16) | (0xcd20 >> 2),
218 0x00000000,
219 (0xbe00 << 16) | (0xcd20 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0x89bc >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0x8900 >> 2),
224 0x00000000,
225 0x3,
226 (0x0e00 << 16) | (0xc130 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc134 >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc1fc >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc208 >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc264 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc268 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc26c >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc270 >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc274 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc278 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc27c >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc280 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc284 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc288 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc28c >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc290 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc294 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc298 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc29c >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc2a0 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0xc2a4 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0xc2a8 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0xc2ac >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0xc2b0 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0x301d0 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0x30238 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0x30250 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0x30254 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0x30258 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0x3025c >> 2),
285 0x00000000,
286 (0x4e00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0x5e00 << 16) | (0xc900 >> 2),
289 0x00000000,
290 (0x6e00 << 16) | (0xc900 >> 2),
291 0x00000000,
292 (0x7e00 << 16) | (0xc900 >> 2),
293 0x00000000,
294 (0x8e00 << 16) | (0xc900 >> 2),
295 0x00000000,
296 (0x9e00 << 16) | (0xc900 >> 2),
297 0x00000000,
298 (0xae00 << 16) | (0xc900 >> 2),
299 0x00000000,
300 (0xbe00 << 16) | (0xc900 >> 2),
301 0x00000000,
302 (0x4e00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0x5e00 << 16) | (0xc904 >> 2),
305 0x00000000,
306 (0x6e00 << 16) | (0xc904 >> 2),
307 0x00000000,
308 (0x7e00 << 16) | (0xc904 >> 2),
309 0x00000000,
310 (0x8e00 << 16) | (0xc904 >> 2),
311 0x00000000,
312 (0x9e00 << 16) | (0xc904 >> 2),
313 0x00000000,
314 (0xae00 << 16) | (0xc904 >> 2),
315 0x00000000,
316 (0xbe00 << 16) | (0xc904 >> 2),
317 0x00000000,
318 (0x4e00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0x5e00 << 16) | (0xc908 >> 2),
321 0x00000000,
322 (0x6e00 << 16) | (0xc908 >> 2),
323 0x00000000,
324 (0x7e00 << 16) | (0xc908 >> 2),
325 0x00000000,
326 (0x8e00 << 16) | (0xc908 >> 2),
327 0x00000000,
328 (0x9e00 << 16) | (0xc908 >> 2),
329 0x00000000,
330 (0xae00 << 16) | (0xc908 >> 2),
331 0x00000000,
332 (0xbe00 << 16) | (0xc908 >> 2),
333 0x00000000,
334 (0x4e00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0x5e00 << 16) | (0xc90c >> 2),
337 0x00000000,
338 (0x6e00 << 16) | (0xc90c >> 2),
339 0x00000000,
340 (0x7e00 << 16) | (0xc90c >> 2),
341 0x00000000,
342 (0x8e00 << 16) | (0xc90c >> 2),
343 0x00000000,
344 (0x9e00 << 16) | (0xc90c >> 2),
345 0x00000000,
346 (0xae00 << 16) | (0xc90c >> 2),
347 0x00000000,
348 (0xbe00 << 16) | (0xc90c >> 2),
349 0x00000000,
350 (0x4e00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0x5e00 << 16) | (0xc910 >> 2),
353 0x00000000,
354 (0x6e00 << 16) | (0xc910 >> 2),
355 0x00000000,
356 (0x7e00 << 16) | (0xc910 >> 2),
357 0x00000000,
358 (0x8e00 << 16) | (0xc910 >> 2),
359 0x00000000,
360 (0x9e00 << 16) | (0xc910 >> 2),
361 0x00000000,
362 (0xae00 << 16) | (0xc910 >> 2),
363 0x00000000,
364 (0xbe00 << 16) | (0xc910 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc99c >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0x9834 >> 2),
369 0x00000000,
370 (0x0000 << 16) | (0x30f00 >> 2),
371 0x00000000,
372 (0x0001 << 16) | (0x30f00 >> 2),
373 0x00000000,
374 (0x0000 << 16) | (0x30f04 >> 2),
375 0x00000000,
376 (0x0001 << 16) | (0x30f04 >> 2),
377 0x00000000,
378 (0x0000 << 16) | (0x30f08 >> 2),
379 0x00000000,
380 (0x0001 << 16) | (0x30f08 >> 2),
381 0x00000000,
382 (0x0000 << 16) | (0x30f0c >> 2),
383 0x00000000,
384 (0x0001 << 16) | (0x30f0c >> 2),
385 0x00000000,
386 (0x0600 << 16) | (0x9b7c >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x8a14 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x8a18 >> 2),
391 0x00000000,
392 (0x0600 << 16) | (0x30a00 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x8bf0 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x8bcc >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x8b24 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x30a04 >> 2),
401 0x00000000,
402 (0x0600 << 16) | (0x30a10 >> 2),
403 0x00000000,
404 (0x0600 << 16) | (0x30a14 >> 2),
405 0x00000000,
406 (0x0600 << 16) | (0x30a18 >> 2),
407 0x00000000,
408 (0x0600 << 16) | (0x30a2c >> 2),
409 0x00000000,
410 (0x0e00 << 16) | (0xc700 >> 2),
411 0x00000000,
412 (0x0e00 << 16) | (0xc704 >> 2),
413 0x00000000,
414 (0x0e00 << 16) | (0xc708 >> 2),
415 0x00000000,
416 (0x0e00 << 16) | (0xc768 >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc770 >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc774 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc778 >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc77c >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc780 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc784 >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc788 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc78c >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc798 >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc79c >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc7a0 >> 2),
439 0x00000000,
440 (0x0400 << 16) | (0xc7a4 >> 2),
441 0x00000000,
442 (0x0400 << 16) | (0xc7a8 >> 2),
443 0x00000000,
444 (0x0400 << 16) | (0xc7ac >> 2),
445 0x00000000,
446 (0x0400 << 16) | (0xc7b0 >> 2),
447 0x00000000,
448 (0x0400 << 16) | (0xc7b4 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x9100 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x3c010 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92a8 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92ac >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92b4 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x92b8 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x92bc >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x92c0 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x92c4 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x92c8 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x92cc >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x92d0 >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0x8c00 >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0x8c04 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0x8c20 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0x8c38 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0x8c3c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xae00 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9604 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac08 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac0c >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac10 >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac14 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac58 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac68 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac6c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac70 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac74 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac78 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0xac7c >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0xac80 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0xac84 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0xac88 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0xac8c >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x970c >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x9714 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x9718 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x971c >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0x4e00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0x5e00 << 16) | (0x31068 >> 2),
531 0x00000000,
532 (0x6e00 << 16) | (0x31068 >> 2),
533 0x00000000,
534 (0x7e00 << 16) | (0x31068 >> 2),
535 0x00000000,
536 (0x8e00 << 16) | (0x31068 >> 2),
537 0x00000000,
538 (0x9e00 << 16) | (0x31068 >> 2),
539 0x00000000,
540 (0xae00 << 16) | (0x31068 >> 2),
541 0x00000000,
542 (0xbe00 << 16) | (0x31068 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xcd10 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xcd14 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0x88b0 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88b4 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88b8 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x88bc >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0x89c0 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x88c4 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x88c8 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x88d0 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x88d4 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x88d8 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x8980 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x30938 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x3093c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x30940 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x89a0 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x30900 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x30904 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x89b4 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x3c210 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x3c214 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x3c218 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8904 >> 2),
591 0x00000000,
592 0x5,
593 (0x0e00 << 16) | (0x8c28 >> 2),
594 (0x0e00 << 16) | (0x8c2c >> 2),
595 (0x0e00 << 16) | (0x8c30 >> 2),
596 (0x0e00 << 16) | (0x8c34 >> 2),
597 (0x0e00 << 16) | (0x9600 >> 2),
598};
599
600static const u32 kalindi_rlc_save_restore_register_list[] =
601{
602 (0x0e00 << 16) | (0xc12c >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc140 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc150 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc15c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xc168 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xc170 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xc204 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xc2b4 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xc2b8 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xc2bc >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xc2c0 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x8228 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x829c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x869c >> 2),
629 0x00000000,
630 (0x0600 << 16) | (0x98f4 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x98f8 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x9900 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0xc260 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x90e8 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c000 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c00c >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8c1c >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0x9700 >> 2),
647 0x00000000,
648 (0x0e00 << 16) | (0xcd20 >> 2),
649 0x00000000,
650 (0x4e00 << 16) | (0xcd20 >> 2),
651 0x00000000,
652 (0x5e00 << 16) | (0xcd20 >> 2),
653 0x00000000,
654 (0x6e00 << 16) | (0xcd20 >> 2),
655 0x00000000,
656 (0x7e00 << 16) | (0xcd20 >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0x89bc >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0x8900 >> 2),
661 0x00000000,
662 0x3,
663 (0x0e00 << 16) | (0xc130 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc134 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc1fc >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc208 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc264 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc268 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc26c >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc270 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc274 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc28c >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc290 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc294 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0xc298 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0xc2a0 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0xc2a4 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0xc2a8 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0xc2ac >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x301d0 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0x30238 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0x30250 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0x30254 >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0x30258 >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0x3025c >> 2),
708 0x00000000,
709 (0x4e00 << 16) | (0xc900 >> 2),
710 0x00000000,
711 (0x5e00 << 16) | (0xc900 >> 2),
712 0x00000000,
713 (0x6e00 << 16) | (0xc900 >> 2),
714 0x00000000,
715 (0x7e00 << 16) | (0xc900 >> 2),
716 0x00000000,
717 (0x4e00 << 16) | (0xc904 >> 2),
718 0x00000000,
719 (0x5e00 << 16) | (0xc904 >> 2),
720 0x00000000,
721 (0x6e00 << 16) | (0xc904 >> 2),
722 0x00000000,
723 (0x7e00 << 16) | (0xc904 >> 2),
724 0x00000000,
725 (0x4e00 << 16) | (0xc908 >> 2),
726 0x00000000,
727 (0x5e00 << 16) | (0xc908 >> 2),
728 0x00000000,
729 (0x6e00 << 16) | (0xc908 >> 2),
730 0x00000000,
731 (0x7e00 << 16) | (0xc908 >> 2),
732 0x00000000,
733 (0x4e00 << 16) | (0xc90c >> 2),
734 0x00000000,
735 (0x5e00 << 16) | (0xc90c >> 2),
736 0x00000000,
737 (0x6e00 << 16) | (0xc90c >> 2),
738 0x00000000,
739 (0x7e00 << 16) | (0xc90c >> 2),
740 0x00000000,
741 (0x4e00 << 16) | (0xc910 >> 2),
742 0x00000000,
743 (0x5e00 << 16) | (0xc910 >> 2),
744 0x00000000,
745 (0x6e00 << 16) | (0xc910 >> 2),
746 0x00000000,
747 (0x7e00 << 16) | (0xc910 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc99c >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x9834 >> 2),
752 0x00000000,
753 (0x0000 << 16) | (0x30f00 >> 2),
754 0x00000000,
755 (0x0000 << 16) | (0x30f04 >> 2),
756 0x00000000,
757 (0x0000 << 16) | (0x30f08 >> 2),
758 0x00000000,
759 (0x0000 << 16) | (0x30f0c >> 2),
760 0x00000000,
761 (0x0600 << 16) | (0x9b7c >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8a14 >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x8a18 >> 2),
766 0x00000000,
767 (0x0600 << 16) | (0x30a00 >> 2),
768 0x00000000,
769 (0x0e00 << 16) | (0x8bf0 >> 2),
770 0x00000000,
771 (0x0e00 << 16) | (0x8bcc >> 2),
772 0x00000000,
773 (0x0e00 << 16) | (0x8b24 >> 2),
774 0x00000000,
775 (0x0e00 << 16) | (0x30a04 >> 2),
776 0x00000000,
777 (0x0600 << 16) | (0x30a10 >> 2),
778 0x00000000,
779 (0x0600 << 16) | (0x30a14 >> 2),
780 0x00000000,
781 (0x0600 << 16) | (0x30a18 >> 2),
782 0x00000000,
783 (0x0600 << 16) | (0x30a2c >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc700 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc704 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc708 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc768 >> 2),
792 0x00000000,
793 (0x0400 << 16) | (0xc770 >> 2),
794 0x00000000,
795 (0x0400 << 16) | (0xc774 >> 2),
796 0x00000000,
797 (0x0400 << 16) | (0xc798 >> 2),
798 0x00000000,
799 (0x0400 << 16) | (0xc79c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x9100 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x3c010 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x8c00 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x8c04 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0x8c20 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x8c38 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x8c3c >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xae00 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x9604 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac08 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac0c >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac10 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac14 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac58 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac68 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac6c >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac70 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac74 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac78 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xac7c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xac80 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xac84 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xac88 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0xac8c >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0x970c >> 2),
850 0x00000000,
851 (0x0e00 << 16) | (0x9714 >> 2),
852 0x00000000,
853 (0x0e00 << 16) | (0x9718 >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x971c >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x31068 >> 2),
858 0x00000000,
859 (0x4e00 << 16) | (0x31068 >> 2),
860 0x00000000,
861 (0x5e00 << 16) | (0x31068 >> 2),
862 0x00000000,
863 (0x6e00 << 16) | (0x31068 >> 2),
864 0x00000000,
865 (0x7e00 << 16) | (0x31068 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xcd10 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xcd14 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x88b0 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88b4 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88b8 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x88bc >> 2),
878 0x00000000,
879 (0x0400 << 16) | (0x89c0 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x88c4 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x88c8 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x88d0 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x88d4 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x88d8 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x8980 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30938 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x3093c >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x30940 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x89a0 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x30900 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x30904 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x89b4 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x3e1fc >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x3c210 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x3c214 >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x3c218 >> 2),
914 0x00000000,
915 (0x0e00 << 16) | (0x8904 >> 2),
916 0x00000000,
917 0x5,
918 (0x0e00 << 16) | (0x8c28 >> 2),
919 (0x0e00 << 16) | (0x8c2c >> 2),
920 (0x0e00 << 16) | (0x8c30 >> 2),
921 (0x0e00 << 16) | (0x8c34 >> 2),
922 (0x0e00 << 16) | (0x9600 >> 2),
923};
924
0aafd313
AD		 925static const u32 bonaire_golden_spm_registers[] =
926{
927 0x30800, 0xe0ffffff, 0xe0000000
928};
929
930static const u32 bonaire_golden_common_registers[] =
931{
932 0xc770, 0xffffffff, 0x00000800,
933 0xc774, 0xffffffff, 0x00000800,
934 0xc798, 0xffffffff, 0x00007fbf,
935 0xc79c, 0xffffffff, 0x00007faf
936};
937
938static const u32 bonaire_golden_registers[] =
939{
940 0x3354, 0x00000333, 0x00000333,
941 0x3350, 0x000c0fc0, 0x00040200,
942 0x9a10, 0x00010000, 0x00058208,
943 0x3c000, 0xffff1fff, 0x00140000,
944 0x3c200, 0xfdfc0fff, 0x00000100,
945 0x3c234, 0x40000000, 0x40000200,
946 0x9830, 0xffffffff, 0x00000000,
947 0x9834, 0xf00fffff, 0x00000400,
948 0x9838, 0x0002021c, 0x00020200,
949 0xc78, 0x00000080, 0x00000000,
950 0x5bb0, 0x000000f0, 0x00000070,
951 0x5bc0, 0xf0311fff, 0x80300000,
952 0x98f8, 0x73773777, 0x12010001,
953 0x350c, 0x00810000, 0x408af000,
954 0x7030, 0x31000111, 0x00000011,
955 0x2f48, 0x73773777, 0x12010001,
956 0x220c, 0x00007fb6, 0x0021a1b1,
957 0x2210, 0x00007fb6, 0x002021b1,
958 0x2180, 0x00007fb6, 0x00002191,
959 0x2218, 0x00007fb6, 0x002121b1,
960 0x221c, 0x00007fb6, 0x002021b1,
961 0x21dc, 0x00007fb6, 0x00002191,
962 0x21e0, 0x00007fb6, 0x00002191,
963 0x3628, 0x0000003f, 0x0000000a,
964 0x362c, 0x0000003f, 0x0000000a,
965 0x2ae4, 0x00073ffe, 0x000022a2,
966 0x240c, 0x000007ff, 0x00000000,
967 0x8a14, 0xf000003f, 0x00000007,
968 0x8bf0, 0x00002001, 0x00000001,
969 0x8b24, 0xffffffff, 0x00ffffff,
970 0x30a04, 0x0000ff0f, 0x00000000,
971 0x28a4c, 0x07ffffff, 0x06000000,
972 0x4d8, 0x00000fff, 0x00000100,
973 0x3e78, 0x00000001, 0x00000002,
974 0x9100, 0x03000000, 0x0362c688,
975 0x8c00, 0x000000ff, 0x00000001,
976 0xe40, 0x00001fff, 0x00001fff,
977 0x9060, 0x0000007f, 0x00000020,
978 0x9508, 0x00010000, 0x00010000,
979 0xac14, 0x000003ff, 0x000000f3,
980 0xac0c, 0xffffffff, 0x00001032
981};
982
983static const u32 bonaire_mgcg_cgcg_init[] =
984{
985 0xc420, 0xffffffff, 0xfffffffc,
986 0x30800, 0xffffffff, 0xe0000000,
987 0x3c2a0, 0xffffffff, 0x00000100,
988 0x3c208, 0xffffffff, 0x00000100,
989 0x3c2c0, 0xffffffff, 0xc0000100,
990 0x3c2c8, 0xffffffff, 0xc0000100,
991 0x3c2c4, 0xffffffff, 0xc0000100,
992 0x55e4, 0xffffffff, 0x00600100,
993 0x3c280, 0xffffffff, 0x00000100,
994 0x3c214, 0xffffffff, 0x06000100,
995 0x3c220, 0xffffffff, 0x00000100,
996 0x3c218, 0xffffffff, 0x06000100,
997 0x3c204, 0xffffffff, 0x00000100,
998 0x3c2e0, 0xffffffff, 0x00000100,
999 0x3c224, 0xffffffff, 0x00000100,
1000 0x3c200, 0xffffffff, 0x00000100,
1001 0x3c230, 0xffffffff, 0x00000100,
1002 0x3c234, 0xffffffff, 0x00000100,
1003 0x3c250, 0xffffffff, 0x00000100,
1004 0x3c254, 0xffffffff, 0x00000100,
1005 0x3c258, 0xffffffff, 0x00000100,
1006 0x3c25c, 0xffffffff, 0x00000100,
1007 0x3c260, 0xffffffff, 0x00000100,
1008 0x3c27c, 0xffffffff, 0x00000100,
1009 0x3c278, 0xffffffff, 0x00000100,
1010 0x3c210, 0xffffffff, 0x06000100,
1011 0x3c290, 0xffffffff, 0x00000100,
1012 0x3c274, 0xffffffff, 0x00000100,
1013 0x3c2b4, 0xffffffff, 0x00000100,
1014 0x3c2b0, 0xffffffff, 0x00000100,
1015 0x3c270, 0xffffffff, 0x00000100,
1016 0x30800, 0xffffffff, 0xe0000000,
1017 0x3c020, 0xffffffff, 0x00010000,
1018 0x3c024, 0xffffffff, 0x00030002,
1019 0x3c028, 0xffffffff, 0x00040007,
1020 0x3c02c, 0xffffffff, 0x00060005,
1021 0x3c030, 0xffffffff, 0x00090008,
1022 0x3c034, 0xffffffff, 0x00010000,
1023 0x3c038, 0xffffffff, 0x00030002,
1024 0x3c03c, 0xffffffff, 0x00040007,
1025 0x3c040, 0xffffffff, 0x00060005,
1026 0x3c044, 0xffffffff, 0x00090008,
1027 0x3c048, 0xffffffff, 0x00010000,
1028 0x3c04c, 0xffffffff, 0x00030002,
1029 0x3c050, 0xffffffff, 0x00040007,
1030 0x3c054, 0xffffffff, 0x00060005,
1031 0x3c058, 0xffffffff, 0x00090008,
1032 0x3c05c, 0xffffffff, 0x00010000,
1033 0x3c060, 0xffffffff, 0x00030002,
1034 0x3c064, 0xffffffff, 0x00040007,
1035 0x3c068, 0xffffffff, 0x00060005,
1036 0x3c06c, 0xffffffff, 0x00090008,
1037 0x3c070, 0xffffffff, 0x00010000,
1038 0x3c074, 0xffffffff, 0x00030002,
1039 0x3c078, 0xffffffff, 0x00040007,
1040 0x3c07c, 0xffffffff, 0x00060005,
1041 0x3c080, 0xffffffff, 0x00090008,
1042 0x3c084, 0xffffffff, 0x00010000,
1043 0x3c088, 0xffffffff, 0x00030002,
1044 0x3c08c, 0xffffffff, 0x00040007,
1045 0x3c090, 0xffffffff, 0x00060005,
1046 0x3c094, 0xffffffff, 0x00090008,
1047 0x3c098, 0xffffffff, 0x00010000,
1048 0x3c09c, 0xffffffff, 0x00030002,
1049 0x3c0a0, 0xffffffff, 0x00040007,
1050 0x3c0a4, 0xffffffff, 0x00060005,
1051 0x3c0a8, 0xffffffff, 0x00090008,
1052 0x3c000, 0xffffffff, 0x96e00200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc424, 0xffffffff, 0x0020003f,
1055 0x38, 0xffffffff, 0x0140001c,
1056 0x3c, 0x000f0000, 0x000f0000,
1057 0x220, 0xffffffff, 0xC060000C,
1058 0x224, 0xc0000fff, 0x00000100,
1059 0xf90, 0xffffffff, 0x00000100,
1060 0xf98, 0x00000101, 0x00000000,
1061 0x20a8, 0xffffffff, 0x00000104,
1062 0x55e4, 0xff000fff, 0x00000100,
1063 0x30cc, 0xc0000fff, 0x00000104,
1064 0xc1e4, 0x00000001, 0x00000001,
1065 0xd00c, 0xff000ff0, 0x00000100,
1066 0xd80c, 0xff000ff0, 0x00000100
1067};
1068
1069static const u32 spectre_golden_spm_registers[] =
1070{
1071 0x30800, 0xe0ffffff, 0xe0000000
1072};
1073
1074static const u32 spectre_golden_common_registers[] =
1075{
1076 0xc770, 0xffffffff, 0x00000800,
1077 0xc774, 0xffffffff, 0x00000800,
1078 0xc798, 0xffffffff, 0x00007fbf,
1079 0xc79c, 0xffffffff, 0x00007faf
1080};
1081
1082static const u32 spectre_golden_registers[] =
1083{
1084 0x3c000, 0xffff1fff, 0x96940200,
1085 0x3c00c, 0xffff0001, 0xff000000,
1086 0x3c200, 0xfffc0fff, 0x00000100,
1087 0x6ed8, 0x00010101, 0x00010000,
1088 0x9834, 0xf00fffff, 0x00000400,
1089 0x9838, 0xfffffffc, 0x00020200,
1090 0x5bb0, 0x000000f0, 0x00000070,
1091 0x5bc0, 0xf0311fff, 0x80300000,
1092 0x98f8, 0x73773777, 0x12010001,
1093 0x9b7c, 0x00ff0000, 0x00fc0000,
1094 0x2f48, 0x73773777, 0x12010001,
1095 0x8a14, 0xf000003f, 0x00000007,
1096 0x8b24, 0xffffffff, 0x00ffffff,
1097 0x28350, 0x3f3f3fff, 0x00000082,
1098 0x28355, 0x0000003f, 0x00000000,
1099 0x3e78, 0x00000001, 0x00000002,
1100 0x913c, 0xffff03df, 0x00000004,
1101 0xc768, 0x00000008, 0x00000008,
1102 0x8c00, 0x000008ff, 0x00000800,
1103 0x9508, 0x00010000, 0x00010000,
1104 0xac0c, 0xffffffff, 0x54763210,
1105 0x214f8, 0x01ff01ff, 0x00000002,
1106 0x21498, 0x007ff800, 0x00200000,
1107 0x2015c, 0xffffffff, 0x00000f40,
1108 0x30934, 0xffffffff, 0x00000001
1109};
1110
1111static const u32 spectre_mgcg_cgcg_init[] =
1112{
1113 0xc420, 0xffffffff, 0xfffffffc,
1114 0x30800, 0xffffffff, 0xe0000000,
1115 0x3c2a0, 0xffffffff, 0x00000100,
1116 0x3c208, 0xffffffff, 0x00000100,
1117 0x3c2c0, 0xffffffff, 0x00000100,
1118 0x3c2c8, 0xffffffff, 0x00000100,
1119 0x3c2c4, 0xffffffff, 0x00000100,
1120 0x55e4, 0xffffffff, 0x00600100,
1121 0x3c280, 0xffffffff, 0x00000100,
1122 0x3c214, 0xffffffff, 0x06000100,
1123 0x3c220, 0xffffffff, 0x00000100,
1124 0x3c218, 0xffffffff, 0x06000100,
1125 0x3c204, 0xffffffff, 0x00000100,
1126 0x3c2e0, 0xffffffff, 0x00000100,
1127 0x3c224, 0xffffffff, 0x00000100,
1128 0x3c200, 0xffffffff, 0x00000100,
1129 0x3c230, 0xffffffff, 0x00000100,
1130 0x3c234, 0xffffffff, 0x00000100,
1131 0x3c250, 0xffffffff, 0x00000100,
1132 0x3c254, 0xffffffff, 0x00000100,
1133 0x3c258, 0xffffffff, 0x00000100,
1134 0x3c25c, 0xffffffff, 0x00000100,
1135 0x3c260, 0xffffffff, 0x00000100,
1136 0x3c27c, 0xffffffff, 0x00000100,
1137 0x3c278, 0xffffffff, 0x00000100,
1138 0x3c210, 0xffffffff, 0x06000100,
1139 0x3c290, 0xffffffff, 0x00000100,
1140 0x3c274, 0xffffffff, 0x00000100,
1141 0x3c2b4, 0xffffffff, 0x00000100,
1142 0x3c2b0, 0xffffffff, 0x00000100,
1143 0x3c270, 0xffffffff, 0x00000100,
1144 0x30800, 0xffffffff, 0xe0000000,
1145 0x3c020, 0xffffffff, 0x00010000,
1146 0x3c024, 0xffffffff, 0x00030002,
1147 0x3c028, 0xffffffff, 0x00040007,
1148 0x3c02c, 0xffffffff, 0x00060005,
1149 0x3c030, 0xffffffff, 0x00090008,
1150 0x3c034, 0xffffffff, 0x00010000,
1151 0x3c038, 0xffffffff, 0x00030002,
1152 0x3c03c, 0xffffffff, 0x00040007,
1153 0x3c040, 0xffffffff, 0x00060005,
1154 0x3c044, 0xffffffff, 0x00090008,
1155 0x3c048, 0xffffffff, 0x00010000,
1156 0x3c04c, 0xffffffff, 0x00030002,
1157 0x3c050, 0xffffffff, 0x00040007,
1158 0x3c054, 0xffffffff, 0x00060005,
1159 0x3c058, 0xffffffff, 0x00090008,
1160 0x3c05c, 0xffffffff, 0x00010000,
1161 0x3c060, 0xffffffff, 0x00030002,
1162 0x3c064, 0xffffffff, 0x00040007,
1163 0x3c068, 0xffffffff, 0x00060005,
1164 0x3c06c, 0xffffffff, 0x00090008,
1165 0x3c070, 0xffffffff, 0x00010000,
1166 0x3c074, 0xffffffff, 0x00030002,
1167 0x3c078, 0xffffffff, 0x00040007,
1168 0x3c07c, 0xffffffff, 0x00060005,
1169 0x3c080, 0xffffffff, 0x00090008,
1170 0x3c084, 0xffffffff, 0x00010000,
1171 0x3c088, 0xffffffff, 0x00030002,
1172 0x3c08c, 0xffffffff, 0x00040007,
1173 0x3c090, 0xffffffff, 0x00060005,
1174 0x3c094, 0xffffffff, 0x00090008,
1175 0x3c098, 0xffffffff, 0x00010000,
1176 0x3c09c, 0xffffffff, 0x00030002,
1177 0x3c0a0, 0xffffffff, 0x00040007,
1178 0x3c0a4, 0xffffffff, 0x00060005,
1179 0x3c0a8, 0xffffffff, 0x00090008,
1180 0x3c0ac, 0xffffffff, 0x00010000,
1181 0x3c0b0, 0xffffffff, 0x00030002,
1182 0x3c0b4, 0xffffffff, 0x00040007,
1183 0x3c0b8, 0xffffffff, 0x00060005,
1184 0x3c0bc, 0xffffffff, 0x00090008,
1185 0x3c000, 0xffffffff, 0x96e00200,
1186 0x8708, 0xffffffff, 0x00900100,
1187 0xc424, 0xffffffff, 0x0020003f,
1188 0x38, 0xffffffff, 0x0140001c,
1189 0x3c, 0x000f0000, 0x000f0000,
1190 0x220, 0xffffffff, 0xC060000C,
1191 0x224, 0xc0000fff, 0x00000100,
1192 0xf90, 0xffffffff, 0x00000100,
1193 0xf98, 0x00000101, 0x00000000,
1194 0x20a8, 0xffffffff, 0x00000104,
1195 0x55e4, 0xff000fff, 0x00000100,
1196 0x30cc, 0xc0000fff, 0x00000104,
1197 0xc1e4, 0x00000001, 0x00000001,
1198 0xd00c, 0xff000ff0, 0x00000100,
1199 0xd80c, 0xff000ff0, 0x00000100
1200};
1201
1202static const u32 kalindi_golden_spm_registers[] =
1203{
1204 0x30800, 0xe0ffffff, 0xe0000000
1205};
1206
1207static const u32 kalindi_golden_common_registers[] =
1208{
1209 0xc770, 0xffffffff, 0x00000800,
1210 0xc774, 0xffffffff, 0x00000800,
1211 0xc798, 0xffffffff, 0x00007fbf,
1212 0xc79c, 0xffffffff, 0x00007faf
1213};
1214
1215static const u32 kalindi_golden_registers[] =
1216{
1217 0x3c000, 0xffffdfff, 0x6e944040,
1218 0x55e4, 0xff607fff, 0xfc000100,
1219 0x3c220, 0xff000fff, 0x00000100,
1220 0x3c224, 0xff000fff, 0x00000100,
1221 0x3c200, 0xfffc0fff, 0x00000100,
1222 0x6ed8, 0x00010101, 0x00010000,
1223 0x9830, 0xffffffff, 0x00000000,
1224 0x9834, 0xf00fffff, 0x00000400,
1225 0x5bb0, 0x000000f0, 0x00000070,
1226 0x5bc0, 0xf0311fff, 0x80300000,
1227 0x98f8, 0x73773777, 0x12010001,
1228 0x98fc, 0xffffffff, 0x00000010,
1229 0x9b7c, 0x00ff0000, 0x00fc0000,
1230 0x8030, 0x00001f0f, 0x0000100a,
1231 0x2f48, 0x73773777, 0x12010001,
1232 0x2408, 0x000fffff, 0x000c007f,
1233 0x8a14, 0xf000003f, 0x00000007,
1234 0x8b24, 0x3fff3fff, 0x00ffcfff,
1235 0x30a04, 0x0000ff0f, 0x00000000,
1236 0x28a4c, 0x07ffffff, 0x06000000,
1237 0x4d8, 0x00000fff, 0x00000100,
1238 0x3e78, 0x00000001, 0x00000002,
1239 0xc768, 0x00000008, 0x00000008,
1240 0x8c00, 0x000000ff, 0x00000003,
1241 0x214f8, 0x01ff01ff, 0x00000002,
1242 0x21498, 0x007ff800, 0x00200000,
1243 0x2015c, 0xffffffff, 0x00000f40,
1244 0x88c4, 0x001f3ae3, 0x00000082,
1245 0x88d4, 0x0000001f, 0x00000010,
1246 0x30934, 0xffffffff, 0x00000000
1247};
1248
1249static const u32 kalindi_mgcg_cgcg_init[] =
1250{
1251 0xc420, 0xffffffff, 0xfffffffc,
1252 0x30800, 0xffffffff, 0xe0000000,
1253 0x3c2a0, 0xffffffff, 0x00000100,
1254 0x3c208, 0xffffffff, 0x00000100,
1255 0x3c2c0, 0xffffffff, 0x00000100,
1256 0x3c2c8, 0xffffffff, 0x00000100,
1257 0x3c2c4, 0xffffffff, 0x00000100,
1258 0x55e4, 0xffffffff, 0x00600100,
1259 0x3c280, 0xffffffff, 0x00000100,
1260 0x3c214, 0xffffffff, 0x06000100,
1261 0x3c220, 0xffffffff, 0x00000100,
1262 0x3c218, 0xffffffff, 0x06000100,
1263 0x3c204, 0xffffffff, 0x00000100,
1264 0x3c2e0, 0xffffffff, 0x00000100,
1265 0x3c224, 0xffffffff, 0x00000100,
1266 0x3c200, 0xffffffff, 0x00000100,
1267 0x3c230, 0xffffffff, 0x00000100,
1268 0x3c234, 0xffffffff, 0x00000100,
1269 0x3c250, 0xffffffff, 0x00000100,
1270 0x3c254, 0xffffffff, 0x00000100,
1271 0x3c258, 0xffffffff, 0x00000100,
1272 0x3c25c, 0xffffffff, 0x00000100,
1273 0x3c260, 0xffffffff, 0x00000100,
1274 0x3c27c, 0xffffffff, 0x00000100,
1275 0x3c278, 0xffffffff, 0x00000100,
1276 0x3c210, 0xffffffff, 0x06000100,
1277 0x3c290, 0xffffffff, 0x00000100,
1278 0x3c274, 0xffffffff, 0x00000100,
1279 0x3c2b4, 0xffffffff, 0x00000100,
1280 0x3c2b0, 0xffffffff, 0x00000100,
1281 0x3c270, 0xffffffff, 0x00000100,
1282 0x30800, 0xffffffff, 0xe0000000,
1283 0x3c020, 0xffffffff, 0x00010000,
1284 0x3c024, 0xffffffff, 0x00030002,
1285 0x3c028, 0xffffffff, 0x00040007,
1286 0x3c02c, 0xffffffff, 0x00060005,
1287 0x3c030, 0xffffffff, 0x00090008,
1288 0x3c034, 0xffffffff, 0x00010000,
1289 0x3c038, 0xffffffff, 0x00030002,
1290 0x3c03c, 0xffffffff, 0x00040007,
1291 0x3c040, 0xffffffff, 0x00060005,
1292 0x3c044, 0xffffffff, 0x00090008,
1293 0x3c000, 0xffffffff, 0x96e00200,
1294 0x8708, 0xffffffff, 0x00900100,
1295 0xc424, 0xffffffff, 0x0020003f,
1296 0x38, 0xffffffff, 0x0140001c,
1297 0x3c, 0x000f0000, 0x000f0000,
1298 0x220, 0xffffffff, 0xC060000C,
1299 0x224, 0xc0000fff, 0x00000100,
1300 0x20a8, 0xffffffff, 0x00000104,
1301 0x55e4, 0xff000fff, 0x00000100,
1302 0x30cc, 0xc0000fff, 0x00000104,
1303 0xc1e4, 0x00000001, 0x00000001,
1304 0xd00c, 0xff000ff0, 0x00000100,
1305 0xd80c, 0xff000ff0, 0x00000100
1306};
1307
8efff337
AD		 1308static const u32 hawaii_golden_spm_registers[] =
1309{
1310 0x30800, 0xe0ffffff, 0xe0000000
1311};
1312
1313static const u32 hawaii_golden_common_registers[] =
1314{
1315 0x30800, 0xffffffff, 0xe0000000,
1316 0x28350, 0xffffffff, 0x3a00161a,
1317 0x28354, 0xffffffff, 0x0000002e,
1318 0x9a10, 0xffffffff, 0x00018208,
1319 0x98f8, 0xffffffff, 0x12011003
1320};
1321
1322static const u32 hawaii_golden_registers[] =
1323{
1324 0x3354, 0x00000333, 0x00000333,
1325 0x9a10, 0x00010000, 0x00058208,
1326 0x9830, 0xffffffff, 0x00000000,
1327 0x9834, 0xf00fffff, 0x00000400,
1328 0x9838, 0x0002021c, 0x00020200,
1329 0xc78, 0x00000080, 0x00000000,
1330 0x5bb0, 0x000000f0, 0x00000070,
1331 0x5bc0, 0xf0311fff, 0x80300000,
1332 0x350c, 0x00810000, 0x408af000,
1333 0x7030, 0x31000111, 0x00000011,
1334 0x2f48, 0x73773777, 0x12010001,
1335 0x2120, 0x0000007f, 0x0000001b,
1336 0x21dc, 0x00007fb6, 0x00002191,
1337 0x3628, 0x0000003f, 0x0000000a,
1338 0x362c, 0x0000003f, 0x0000000a,
1339 0x2ae4, 0x00073ffe, 0x000022a2,
1340 0x240c, 0x000007ff, 0x00000000,
1341 0x8bf0, 0x00002001, 0x00000001,
1342 0x8b24, 0xffffffff, 0x00ffffff,
1343 0x30a04, 0x0000ff0f, 0x00000000,
1344 0x28a4c, 0x07ffffff, 0x06000000,
1345 0x3e78, 0x00000001, 0x00000002,
1346 0xc768, 0x00000008, 0x00000008,
1347 0xc770, 0x00000f00, 0x00000800,
1348 0xc774, 0x00000f00, 0x00000800,
1349 0xc798, 0x00ffffff, 0x00ff7fbf,
1350 0xc79c, 0x00ffffff, 0x00ff7faf,
1351 0x8c00, 0x000000ff, 0x00000800,
1352 0xe40, 0x00001fff, 0x00001fff,
1353 0x9060, 0x0000007f, 0x00000020,
1354 0x9508, 0x00010000, 0x00010000,
1355 0xae00, 0x00100000, 0x000ff07c,
1356 0xac14, 0x000003ff, 0x0000000f,
1357 0xac10, 0xffffffff, 0x7564fdec,
1358 0xac0c, 0xffffffff, 0x3120b9a8,
1359 0xac08, 0x20000000, 0x0f9c0000
1360};
1361
1362static const u32 hawaii_mgcg_cgcg_init[] =
1363{
1364 0xc420, 0xffffffff, 0xfffffffd,
1365 0x30800, 0xffffffff, 0xe0000000,
1366 0x3c2a0, 0xffffffff, 0x00000100,
1367 0x3c208, 0xffffffff, 0x00000100,
1368 0x3c2c0, 0xffffffff, 0x00000100,
1369 0x3c2c8, 0xffffffff, 0x00000100,
1370 0x3c2c4, 0xffffffff, 0x00000100,
1371 0x55e4, 0xffffffff, 0x00200100,
1372 0x3c280, 0xffffffff, 0x00000100,
1373 0x3c214, 0xffffffff, 0x06000100,
1374 0x3c220, 0xffffffff, 0x00000100,
1375 0x3c218, 0xffffffff, 0x06000100,
1376 0x3c204, 0xffffffff, 0x00000100,
1377 0x3c2e0, 0xffffffff, 0x00000100,
1378 0x3c224, 0xffffffff, 0x00000100,
1379 0x3c200, 0xffffffff, 0x00000100,
1380 0x3c230, 0xffffffff, 0x00000100,
1381 0x3c234, 0xffffffff, 0x00000100,
1382 0x3c250, 0xffffffff, 0x00000100,
1383 0x3c254, 0xffffffff, 0x00000100,
1384 0x3c258, 0xffffffff, 0x00000100,
1385 0x3c25c, 0xffffffff, 0x00000100,
1386 0x3c260, 0xffffffff, 0x00000100,
1387 0x3c27c, 0xffffffff, 0x00000100,
1388 0x3c278, 0xffffffff, 0x00000100,
1389 0x3c210, 0xffffffff, 0x06000100,
1390 0x3c290, 0xffffffff, 0x00000100,
1391 0x3c274, 0xffffffff, 0x00000100,
1392 0x3c2b4, 0xffffffff, 0x00000100,
1393 0x3c2b0, 0xffffffff, 0x00000100,
1394 0x3c270, 0xffffffff, 0x00000100,
1395 0x30800, 0xffffffff, 0xe0000000,
1396 0x3c020, 0xffffffff, 0x00010000,
1397 0x3c024, 0xffffffff, 0x00030002,
1398 0x3c028, 0xffffffff, 0x00040007,
1399 0x3c02c, 0xffffffff, 0x00060005,
1400 0x3c030, 0xffffffff, 0x00090008,
1401 0x3c034, 0xffffffff, 0x00010000,
1402 0x3c038, 0xffffffff, 0x00030002,
1403 0x3c03c, 0xffffffff, 0x00040007,
1404 0x3c040, 0xffffffff, 0x00060005,
1405 0x3c044, 0xffffffff, 0x00090008,
1406 0x3c048, 0xffffffff, 0x00010000,
1407 0x3c04c, 0xffffffff, 0x00030002,
1408 0x3c050, 0xffffffff, 0x00040007,
1409 0x3c054, 0xffffffff, 0x00060005,
1410 0x3c058, 0xffffffff, 0x00090008,
1411 0x3c05c, 0xffffffff, 0x00010000,
1412 0x3c060, 0xffffffff, 0x00030002,
1413 0x3c064, 0xffffffff, 0x00040007,
1414 0x3c068, 0xffffffff, 0x00060005,
1415 0x3c06c, 0xffffffff, 0x00090008,
1416 0x3c070, 0xffffffff, 0x00010000,
1417 0x3c074, 0xffffffff, 0x00030002,
1418 0x3c078, 0xffffffff, 0x00040007,
1419 0x3c07c, 0xffffffff, 0x00060005,
1420 0x3c080, 0xffffffff, 0x00090008,
1421 0x3c084, 0xffffffff, 0x00010000,
1422 0x3c088, 0xffffffff, 0x00030002,
1423 0x3c08c, 0xffffffff, 0x00040007,
1424 0x3c090, 0xffffffff, 0x00060005,
1425 0x3c094, 0xffffffff, 0x00090008,
1426 0x3c098, 0xffffffff, 0x00010000,
1427 0x3c09c, 0xffffffff, 0x00030002,
1428 0x3c0a0, 0xffffffff, 0x00040007,
1429 0x3c0a4, 0xffffffff, 0x00060005,
1430 0x3c0a8, 0xffffffff, 0x00090008,
1431 0x3c0ac, 0xffffffff, 0x00010000,
1432 0x3c0b0, 0xffffffff, 0x00030002,
1433 0x3c0b4, 0xffffffff, 0x00040007,
1434 0x3c0b8, 0xffffffff, 0x00060005,
1435 0x3c0bc, 0xffffffff, 0x00090008,
1436 0x3c0c0, 0xffffffff, 0x00010000,
1437 0x3c0c4, 0xffffffff, 0x00030002,
1438 0x3c0c8, 0xffffffff, 0x00040007,
1439 0x3c0cc, 0xffffffff, 0x00060005,
1440 0x3c0d0, 0xffffffff, 0x00090008,
1441 0x3c0d4, 0xffffffff, 0x00010000,
1442 0x3c0d8, 0xffffffff, 0x00030002,
1443 0x3c0dc, 0xffffffff, 0x00040007,
1444 0x3c0e0, 0xffffffff, 0x00060005,
1445 0x3c0e4, 0xffffffff, 0x00090008,
1446 0x3c0e8, 0xffffffff, 0x00010000,
1447 0x3c0ec, 0xffffffff, 0x00030002,
1448 0x3c0f0, 0xffffffff, 0x00040007,
1449 0x3c0f4, 0xffffffff, 0x00060005,
1450 0x3c0f8, 0xffffffff, 0x00090008,
1451 0xc318, 0xffffffff, 0x00020200,
1452 0x3350, 0xffffffff, 0x00000200,
1453 0x15c0, 0xffffffff, 0x00000400,
1454 0x55e8, 0xffffffff, 0x00000000,
1455 0x2f50, 0xffffffff, 0x00000902,
1456 0x3c000, 0xffffffff, 0x96940200,
1457 0x8708, 0xffffffff, 0x00900100,
1458 0xc424, 0xffffffff, 0x0020003f,
1459 0x38, 0xffffffff, 0x0140001c,
1460 0x3c, 0x000f0000, 0x000f0000,
1461 0x220, 0xffffffff, 0xc060000c,
1462 0x224, 0xc0000fff, 0x00000100,
1463 0xf90, 0xffffffff, 0x00000100,
1464 0xf98, 0x00000101, 0x00000000,
1465 0x20a8, 0xffffffff, 0x00000104,
1466 0x55e4, 0xff000fff, 0x00000100,
1467 0x30cc, 0xc0000fff, 0x00000104,
1468 0xc1e4, 0x00000001, 0x00000001,
1469 0xd00c, 0xff000ff0, 0x00000100,
1470 0xd80c, 0xff000ff0, 0x00000100
1471};
1472
0aafd313
AD		 1473static void cik_init_golden_registers(struct radeon_device *rdev)
1474{
1475 switch (rdev->family) {
1476 case CHIP_BONAIRE:
1477 radeon_program_register_sequence(rdev,
1478 bonaire_mgcg_cgcg_init,
1479 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1480 radeon_program_register_sequence(rdev,
1481 bonaire_golden_registers,
1482 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_common_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_spm_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1489 break;
1490 case CHIP_KABINI:
1491 radeon_program_register_sequence(rdev,
1492 kalindi_mgcg_cgcg_init,
1493 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1494 radeon_program_register_sequence(rdev,
1495 kalindi_golden_registers,
1496 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_common_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_spm_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1503 break;
1504 case CHIP_KAVERI:
1505 radeon_program_register_sequence(rdev,
1506 spectre_mgcg_cgcg_init,
1507 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1508 radeon_program_register_sequence(rdev,
1509 spectre_golden_registers,
1510 (const u32)ARRAY_SIZE(spectre_golden_registers));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_common_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_spm_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1517 break;
8efff337
AD		 1518 case CHIP_HAWAII:
1519 radeon_program_register_sequence(rdev,
1520 hawaii_mgcg_cgcg_init,
1521 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1522 radeon_program_register_sequence(rdev,
1523 hawaii_golden_registers,
1524 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_common_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_spm_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1531 break;
0aafd313
AD		 1532 default:
1533 break;
1534 }
1535}
1536
2c67912c
AD		 1537/**
1538 * cik_get_xclk - get the xclk
1539 *
1540 * @rdev: radeon_device pointer
1541 *
1542 * Returns the reference clock used by the gfx engine
1543 * (CIK).
1544 */
1545u32 cik_get_xclk(struct radeon_device *rdev)
1546{
1547 u32 reference_clock = rdev->clock.spll.reference_freq;
1548
1549 if (rdev->flags & RADEON_IS_IGP) {
1550 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1551 return reference_clock / 2;
1552 } else {
1553 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1554 return reference_clock / 4;
1555 }
1556 return reference_clock;
1557}
1558
75efdee1
AD		 1559/**
1560 * cik_mm_rdoorbell - read a doorbell dword
1561 *
1562 * @rdev: radeon_device pointer
1563 * @offset: byte offset into the aperture
1564 *
1565 * Returns the value in the doorbell aperture at the
1566 * requested offset (CIK).
1567 */
1568u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
1569{
1570 if (offset < rdev->doorbell.size) {
1571 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
1572 } else {
1573 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
1574 return 0;
1575 }
1576}
1577
1578/**
1579 * cik_mm_wdoorbell - write a doorbell dword
1580 *
1581 * @rdev: radeon_device pointer
1582 * @offset: byte offset into the aperture
1583 * @v: value to write
1584 *
1585 * Writes @v to the doorbell aperture at the
1586 * requested offset (CIK).
1587 */
1588void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
1589{
1590 if (offset < rdev->doorbell.size) {
1591 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
1592 } else {
1593 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
1594 }
1595}
1596
bc8273fe
AD		 1597#define BONAIRE_IO_MC_REGS_SIZE 36
1598
1599static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1600{
1601 {0x00000070, 0x04400000},
1602 {0x00000071, 0x80c01803},
1603 {0x00000072, 0x00004004},
1604 {0x00000073, 0x00000100},
1605 {0x00000074, 0x00ff0000},
1606 {0x00000075, 0x34000000},
1607 {0x00000076, 0x08000014},
1608 {0x00000077, 0x00cc08ec},
1609 {0x00000078, 0x00000400},
1610 {0x00000079, 0x00000000},
1611 {0x0000007a, 0x04090000},
1612 {0x0000007c, 0x00000000},
1613 {0x0000007e, 0x4408a8e8},
1614 {0x0000007f, 0x00000304},
1615 {0x00000080, 0x00000000},
1616 {0x00000082, 0x00000001},
1617 {0x00000083, 0x00000002},
1618 {0x00000084, 0xf3e4f400},
1619 {0x00000085, 0x052024e3},
1620 {0x00000087, 0x00000000},
1621 {0x00000088, 0x01000000},
1622 {0x0000008a, 0x1c0a0000},
1623 {0x0000008b, 0xff010000},
1624 {0x0000008d, 0xffffefff},
1625 {0x0000008e, 0xfff3efff},
1626 {0x0000008f, 0xfff3efbf},
1627 {0x00000092, 0xf7ffffff},
1628 {0x00000093, 0xffffff7f},
1629 {0x00000095, 0x00101101},
1630 {0x00000096, 0x00000fff},
1631 {0x00000097, 0x00116fff},
1632 {0x00000098, 0x60010000},
1633 {0x00000099, 0x10010000},
1634 {0x0000009a, 0x00006000},
1635 {0x0000009b, 0x00001000},
1636 {0x0000009f, 0x00b48000}
1637};
1638
d4775655
AD		 1639#define HAWAII_IO_MC_REGS_SIZE 22
1640
1641static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1642{
1643 {0x0000007d, 0x40000000},
1644 {0x0000007e, 0x40180304},
1645 {0x0000007f, 0x0000ff00},
1646 {0x00000081, 0x00000000},
1647 {0x00000083, 0x00000800},
1648 {0x00000086, 0x00000000},
1649 {0x00000087, 0x00000100},
1650 {0x00000088, 0x00020100},
1651 {0x00000089, 0x00000000},
1652 {0x0000008b, 0x00040000},
1653 {0x0000008c, 0x00000100},
1654 {0x0000008e, 0xff010000},
1655 {0x00000090, 0xffffefff},
1656 {0x00000091, 0xfff3efff},
1657 {0x00000092, 0xfff3efbf},
1658 {0x00000093, 0xf7ffffff},
1659 {0x00000094, 0xffffff7f},
1660 {0x00000095, 0x00000fff},
1661 {0x00000096, 0x00116fff},
1662 {0x00000097, 0x60010000},
1663 {0x00000098, 0x10010000},
1664 {0x0000009f, 0x00c79000}
1665};
1666
1667
b556b12e
AD		 1668/**
1669 * cik_srbm_select - select specific register instances
1670 *
1671 * @rdev: radeon_device pointer
1672 * @me: selected ME (micro engine)
1673 * @pipe: pipe
1674 * @queue: queue
1675 * @vmid: VMID
1676 *
1677 * Switches the currently active registers instances. Some
1678 * registers are instanced per VMID, others are instanced per
1679 * me/pipe/queue combination.
1680 */
1681static void cik_srbm_select(struct radeon_device *rdev,
1682 u32 me, u32 pipe, u32 queue, u32 vmid)
1683{
1684 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1685 MEID(me & 0x3) |
1686 VMID(vmid & 0xf) |
1687 QUEUEID(queue & 0x7));
1688 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1689}
1690
bc8273fe
AD		 1691/* ucode loading */
1692/**
1693 * ci_mc_load_microcode - load MC ucode into the hw
1694 *
1695 * @rdev: radeon_device pointer
1696 *
1697 * Load the GDDR MC ucode into the hw (CIK).
1698 * Returns 0 on success, error on failure.
1699 */
1700static int ci_mc_load_microcode(struct radeon_device *rdev)
1701{
1702 const __be32 *fw_data;
1703 u32 running, blackout = 0;
1704 u32 *io_mc_regs;
1705 int i, ucode_size, regs_size;
1706
1707 if (!rdev->mc_fw)
1708 return -EINVAL;
1709
1710 switch (rdev->family) {
1711 case CHIP_BONAIRE:
bc8273fe
AD		 1712 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1713 ucode_size = CIK_MC_UCODE_SIZE;
1714 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1715 break;
d4775655
AD		 1716 case CHIP_HAWAII:
1717 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1718 ucode_size = HAWAII_MC_UCODE_SIZE;
1719 regs_size = HAWAII_IO_MC_REGS_SIZE;
1720 break;
1721 default:
1722 return -EINVAL;
bc8273fe
AD		 1723 }
1724
1725 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1726
1727 if (running == 0) {
1728 if (running) {
1729 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1730 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1731 }
1732
1733 /* reset the engine and set to writable */
1734 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1736
1737 /* load mc io regs */
1738 for (i = 0; i < regs_size; i++) {
1739 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1740 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1741 }
1742 /* load the MC ucode */
1743 fw_data = (const __be32 *)rdev->mc_fw->data;
1744 for (i = 0; i < ucode_size; i++)
1745 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1746
1747 /* put the engine back into the active state */
1748 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1751
1752 /* wait for training to complete */
1753 for (i = 0; i < rdev->usec_timeout; i++) {
1754 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1755 break;
1756 udelay(1);
1757 }
1758 for (i = 0; i < rdev->usec_timeout; i++) {
1759 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1760 break;
1761 udelay(1);
1762 }
1763
1764 if (running)
1765 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1766 }
1767
1768 return 0;
1769}
1770
02c81327
AD		 1771/**
1772 * cik_init_microcode - load ucode images from disk
1773 *
1774 * @rdev: radeon_device pointer
1775 *
1776 * Use the firmware interface to load the ucode images into
1777 * the driver (not loaded into hw).
1778 * Returns 0 on success, error on failure.
1779 */
1780static int cik_init_microcode(struct radeon_device *rdev)
1781{
02c81327
AD		 1782 const char *chip_name;
1783 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655
AD		 1784 mec_req_size, rlc_req_size, mc_req_size = 0,
1785 sdma_req_size, smc_req_size = 0;
02c81327
AD		 1786 char fw_name[30];
1787 int err;
1788
1789 DRM_DEBUG("\n");
1790
02c81327
AD		 1791 switch (rdev->family) {
1792 case CHIP_BONAIRE:
1793 chip_name = "BONAIRE";
1794 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1795 me_req_size = CIK_ME_UCODE_SIZE * 4;
1796 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1797 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1798 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1799 mc_req_size = CIK_MC_UCODE_SIZE * 4;
21a93e13 1800 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 1801 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
02c81327 1802 break;
d4775655
AD		 1803 case CHIP_HAWAII:
1804 chip_name = "HAWAII";
1805 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1806 me_req_size = CIK_ME_UCODE_SIZE * 4;
1807 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1808 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1809 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1810 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1811 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1812 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1813 break;
02c81327
AD		 1814 case CHIP_KAVERI:
1815 chip_name = "KAVERI";
1816 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1817 me_req_size = CIK_ME_UCODE_SIZE * 4;
1818 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1819 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1820 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 1821 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1822 break;
1823 case CHIP_KABINI:
1824 chip_name = "KABINI";
1825 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1826 me_req_size = CIK_ME_UCODE_SIZE * 4;
1827 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1828 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1829 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 1830 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1831 break;
1832 default: BUG();
1833 }
1834
1835 DRM_INFO("Loading %s Microcode\n", chip_name);
1836
1837 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1838 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
02c81327
AD		 1839 if (err)
1840 goto out;
1841 if (rdev->pfp_fw->size != pfp_req_size) {
1842 printk(KERN_ERR
1843 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1844 rdev->pfp_fw->size, fw_name);
1845 err = -EINVAL;
1846 goto out;
1847 }
1848
1849 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1850 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
02c81327
AD		 1851 if (err)
1852 goto out;
1853 if (rdev->me_fw->size != me_req_size) {
1854 printk(KERN_ERR
1855 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1856 rdev->me_fw->size, fw_name);
1857 err = -EINVAL;
1858 }
1859
1860 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1861 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
02c81327
AD		 1862 if (err)
1863 goto out;
1864 if (rdev->ce_fw->size != ce_req_size) {
1865 printk(KERN_ERR
1866 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1867 rdev->ce_fw->size, fw_name);
1868 err = -EINVAL;
1869 }
1870
1871 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
0a168933 1872 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
02c81327
AD		 1873 if (err)
1874 goto out;
1875 if (rdev->mec_fw->size != mec_req_size) {
1876 printk(KERN_ERR
1877 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1878 rdev->mec_fw->size, fw_name);
1879 err = -EINVAL;
1880 }
1881
1882 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
0a168933 1883 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
02c81327
AD		 1884 if (err)
1885 goto out;
1886 if (rdev->rlc_fw->size != rlc_req_size) {
1887 printk(KERN_ERR
1888 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1889 rdev->rlc_fw->size, fw_name);
1890 err = -EINVAL;
1891 }
1892
21a93e13 1893 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
0a168933 1894 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
21a93e13
AD		 1895 if (err)
1896 goto out;
1897 if (rdev->sdma_fw->size != sdma_req_size) {
1898 printk(KERN_ERR
1899 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1900 rdev->sdma_fw->size, fw_name);
1901 err = -EINVAL;
1902 }
1903
cc8dbbb4 1904 /* No SMC, MC ucode on APUs */
02c81327
AD		 1905 if (!(rdev->flags & RADEON_IS_IGP)) {
1906 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1907 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
02c81327
AD		 1908 if (err)
1909 goto out;
1910 if (rdev->mc_fw->size != mc_req_size) {
1911 printk(KERN_ERR
1912 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1913 rdev->mc_fw->size, fw_name);
1914 err = -EINVAL;
1915 }
cc8dbbb4
AD		 1916
1917 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1918 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1919 if (err) {
1920 printk(KERN_ERR
1921 "smc: error loading firmware \"%s\"\n",
1922 fw_name);
1923 release_firmware(rdev->smc_fw);
1924 rdev->smc_fw = NULL;
d30d71e8 1925 err = 0;
cc8dbbb4
AD		 1926 } else if (rdev->smc_fw->size != smc_req_size) {
1927 printk(KERN_ERR
1928 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1929 rdev->smc_fw->size, fw_name);
1930 err = -EINVAL;
1931 }
02c81327
AD		 1932 }
1933
1934out:
02c81327
AD		 1935 if (err) {
1936 if (err != -EINVAL)
1937 printk(KERN_ERR
1938 "cik_cp: Failed to load firmware \"%s\"\n",
1939 fw_name);
1940 release_firmware(rdev->pfp_fw);
1941 rdev->pfp_fw = NULL;
1942 release_firmware(rdev->me_fw);
1943 rdev->me_fw = NULL;
1944 release_firmware(rdev->ce_fw);
1945 rdev->ce_fw = NULL;
1946 release_firmware(rdev->rlc_fw);
1947 rdev->rlc_fw = NULL;
1948 release_firmware(rdev->mc_fw);
1949 rdev->mc_fw = NULL;
cc8dbbb4
AD		 1950 release_firmware(rdev->smc_fw);
1951 rdev->smc_fw = NULL;
02c81327
AD		 1952 }
1953 return err;
1954}
1955
8cc1a532
AD		 1956/*
1957 * Core functions
1958 */
1959/**
1960 * cik_tiling_mode_table_init - init the hw tiling table
1961 *
1962 * @rdev: radeon_device pointer
1963 *
1964 * Starting with SI, the tiling setup is done globally in a
1965 * set of 32 tiling modes. Rather than selecting each set of
1966 * parameters per surface as on older asics, we just select
1967 * which index in the tiling table we want to use, and the
1968 * surface uses those parameters (CIK).
1969 */
1970static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1971{
1972 const u32 num_tile_mode_states = 32;
1973 const u32 num_secondary_tile_mode_states = 16;
1974 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1975 u32 num_pipe_configs;
1976 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1977 rdev->config.cik.max_shader_engines;
1978
1979 switch (rdev->config.cik.mem_row_size_in_kb) {
1980 case 1:
1981 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1982 break;
1983 case 2:
1984 default:
1985 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1986 break;
1987 case 4:
1988 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1989 break;
1990 }
1991
1992 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1993 if (num_pipe_configs > 8)
21e438af 1994 num_pipe_configs = 16;
8cc1a532 1995
21e438af
AD		 1996 if (num_pipe_configs == 16) {
1997 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1998 switch (reg_offset) {
1999 case 0:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2004 break;
2005 case 1:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2010 break;
2011 case 2:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2016 break;
2017 case 3:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2020 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2021 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2022 break;
2023 case 4:
2024 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2025 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2026 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2027 TILE_SPLIT(split_equal_to_row_size));
2028 break;
2029 case 5:
2030 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2031 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2032 break;
2033 case 6:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2038 break;
2039 case 7:
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2042 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043 TILE_SPLIT(split_equal_to_row_size));
2044 break;
2045 case 8:
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2047 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2048 break;
2049 case 9:
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2052 break;
2053 case 10:
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2058 break;
2059 case 11:
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2062 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2063 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2064 break;
2065 case 12:
2066 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2067 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2068 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2069 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2070 break;
2071 case 13:
2072 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2073 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2074 break;
2075 case 14:
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2080 break;
2081 case 16:
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2084 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2085 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2086 break;
2087 case 17:
2088 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2089 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2090 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092 break;
2093 case 27:
2094 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2095 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2096 break;
2097 case 28:
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102 break;
2103 case 29:
2104 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2106 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2107 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2108 break;
2109 case 30:
2110 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2111 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2112 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2113 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2114 break;
2115 default:
2116 gb_tile_moden = 0;
2117 break;
2118 }
2119 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2120 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2121 }
2122 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2123 switch (reg_offset) {
2124 case 0:
2125 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2128 NUM_BANKS(ADDR_SURF_16_BANK));
2129 break;
2130 case 1:
2131 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2134 NUM_BANKS(ADDR_SURF_16_BANK));
2135 break;
2136 case 2:
2137 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140 NUM_BANKS(ADDR_SURF_16_BANK));
2141 break;
2142 case 3:
2143 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146 NUM_BANKS(ADDR_SURF_16_BANK));
2147 break;
2148 case 4:
2149 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152 NUM_BANKS(ADDR_SURF_8_BANK));
2153 break;
2154 case 5:
2155 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2158 NUM_BANKS(ADDR_SURF_4_BANK));
2159 break;
2160 case 6:
2161 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2162 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2163 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2164 NUM_BANKS(ADDR_SURF_2_BANK));
2165 break;
2166 case 8:
2167 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2168 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2169 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2170 NUM_BANKS(ADDR_SURF_16_BANK));
2171 break;
2172 case 9:
2173 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2174 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2175 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2176 NUM_BANKS(ADDR_SURF_16_BANK));
2177 break;
2178 case 10:
2179 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2180 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2181 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2182 NUM_BANKS(ADDR_SURF_16_BANK));
2183 break;
2184 case 11:
2185 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2188 NUM_BANKS(ADDR_SURF_8_BANK));
2189 break;
2190 case 12:
2191 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2194 NUM_BANKS(ADDR_SURF_4_BANK));
2195 break;
2196 case 13:
2197 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2200 NUM_BANKS(ADDR_SURF_2_BANK));
2201 break;
2202 case 14:
2203 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2204 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2205 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2206 NUM_BANKS(ADDR_SURF_2_BANK));
2207 break;
2208 default:
2209 gb_tile_moden = 0;
2210 break;
2211 }
2212 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2213 }
2214 } else if (num_pipe_configs == 8) {
8cc1a532
AD		 2215 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2216 switch (reg_offset) {
2217 case 0:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2222 break;
2223 case 1:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2227 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2228 break;
2229 case 2:
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2234 break;
2235 case 3:
2236 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2237 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2238 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2239 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2240 break;
2241 case 4:
2242 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2243 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2244 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2245 TILE_SPLIT(split_equal_to_row_size));
2246 break;
2247 case 5:
2248 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2249 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2250 break;
2251 case 6:
2252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2256 break;
2257 case 7:
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2261 TILE_SPLIT(split_equal_to_row_size));
2262 break;
2263 case 8:
2264 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2266 break;
2267 case 9:
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2270 break;
2271 case 10:
2272 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2274 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2275 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2276 break;
2277 case 11:
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2279 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2282 break;
2283 case 12:
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2286 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2287 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2288 break;
2289 case 13:
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2291 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2292 break;
2293 case 14:
2294 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2296 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2297 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2298 break;
2299 case 16:
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2301 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2304 break;
2305 case 17:
2306 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2307 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2310 break;
2311 case 27:
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2314 break;
2315 case 28:
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2320 break;
2321 case 29:
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2325 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2326 break;
2327 case 30:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2332 break;
2333 default:
2334 gb_tile_moden = 0;
2335 break;
2336 }
39aee490 2337 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2338 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2339 }
2340 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2341 switch (reg_offset) {
2342 case 0:
2343 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2346 NUM_BANKS(ADDR_SURF_16_BANK));
2347 break;
2348 case 1:
2349 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352 NUM_BANKS(ADDR_SURF_16_BANK));
2353 break;
2354 case 2:
2355 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2358 NUM_BANKS(ADDR_SURF_16_BANK));
2359 break;
2360 case 3:
2361 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2364 NUM_BANKS(ADDR_SURF_16_BANK));
2365 break;
2366 case 4:
2367 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370 NUM_BANKS(ADDR_SURF_8_BANK));
2371 break;
2372 case 5:
2373 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2376 NUM_BANKS(ADDR_SURF_4_BANK));
2377 break;
2378 case 6:
2379 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2381 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2382 NUM_BANKS(ADDR_SURF_2_BANK));
2383 break;
2384 case 8:
2385 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2388 NUM_BANKS(ADDR_SURF_16_BANK));
2389 break;
2390 case 9:
2391 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2392 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2393 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2394 NUM_BANKS(ADDR_SURF_16_BANK));
2395 break;
2396 case 10:
2397 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2398 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2399 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2400 NUM_BANKS(ADDR_SURF_16_BANK));
2401 break;
2402 case 11:
2403 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2406 NUM_BANKS(ADDR_SURF_16_BANK));
2407 break;
2408 case 12:
2409 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2412 NUM_BANKS(ADDR_SURF_8_BANK));
2413 break;
2414 case 13:
2415 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2418 NUM_BANKS(ADDR_SURF_4_BANK));
2419 break;
2420 case 14:
2421 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2424 NUM_BANKS(ADDR_SURF_2_BANK));
2425 break;
2426 default:
2427 gb_tile_moden = 0;
2428 break;
2429 }
2430 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2431 }
2432 } else if (num_pipe_configs == 4) {
2433 if (num_rbs == 4) {
2434 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2435 switch (reg_offset) {
2436 case 0:
2437 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2438 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2439 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2440 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2441 break;
2442 case 1:
2443 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2444 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2445 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2446 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2447 break;
2448 case 2:
2449 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2450 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2451 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2452 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2453 break;
2454 case 3:
2455 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2456 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2457 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2458 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2459 break;
2460 case 4:
2461 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2462 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2463 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2464 TILE_SPLIT(split_equal_to_row_size));
2465 break;
2466 case 5:
2467 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2468 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2469 break;
2470 case 6:
2471 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2472 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2473 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2474 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2475 break;
2476 case 7:
2477 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2478 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2479 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2480 TILE_SPLIT(split_equal_to_row_size));
2481 break;
2482 case 8:
2483 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2484 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2485 break;
2486 case 9:
2487 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2488 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2489 break;
2490 case 10:
2491 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2492 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2493 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2494 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2495 break;
2496 case 11:
2497 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2498 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2499 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2500 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2501 break;
2502 case 12:
2503 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2504 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2505 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2506 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2507 break;
2508 case 13:
2509 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2510 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2511 break;
2512 case 14:
2513 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2515 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2516 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2517 break;
2518 case 16:
2519 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2520 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2521 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2522 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2523 break;
2524 case 17:
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2528 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2529 break;
2530 case 27:
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2533 break;
2534 case 28:
2535 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2536 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2537 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2538 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2539 break;
2540 case 29:
2541 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2542 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2543 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2544 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2545 break;
2546 case 30:
2547 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2548 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2549 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2550 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2551 break;
2552 default:
2553 gb_tile_moden = 0;
2554 break;
2555 }
39aee490 2556 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2557 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2558 }
2559 } else if (num_rbs < 4) {
2560 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2561 switch (reg_offset) {
2562 case 0:
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2566 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2567 break;
2568 case 1:
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2570 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2571 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2572 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2573 break;
2574 case 2:
2575 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2576 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2578 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2579 break;
2580 case 3:
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2584 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2585 break;
2586 case 4:
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2590 TILE_SPLIT(split_equal_to_row_size));
2591 break;
2592 case 5:
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2595 break;
2596 case 6:
2597 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2598 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2599 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2600 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2601 break;
2602 case 7:
2603 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2604 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2605 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2606 TILE_SPLIT(split_equal_to_row_size));
2607 break;
2608 case 8:
2609 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2610 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2611 break;
2612 case 9:
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2614 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2615 break;
2616 case 10:
2617 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2618 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2619 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2620 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2621 break;
2622 case 11:
2623 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2624 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2625 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2626 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2627 break;
2628 case 12:
2629 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2630 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2631 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2632 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2633 break;
2634 case 13:
2635 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2636 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2637 break;
2638 case 14:
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643 break;
2644 case 16:
2645 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2646 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2647 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2648 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2649 break;
2650 case 17:
2651 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2652 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2653 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2654 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2655 break;
2656 case 27:
2657 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2658 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2659 break;
2660 case 28:
2661 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2662 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2663 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2664 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2665 break;
2666 case 29:
2667 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2668 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2669 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2670 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2671 break;
2672 case 30:
2673 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2674 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2675 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2676 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2677 break;
2678 default:
2679 gb_tile_moden = 0;
2680 break;
2681 }
39aee490 2682 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2683 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2684 }
2685 }
2686 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2687 switch (reg_offset) {
2688 case 0:
2689 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2690 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2691 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2692 NUM_BANKS(ADDR_SURF_16_BANK));
2693 break;
2694 case 1:
2695 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2696 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2697 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2698 NUM_BANKS(ADDR_SURF_16_BANK));
2699 break;
2700 case 2:
2701 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2703 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2704 NUM_BANKS(ADDR_SURF_16_BANK));
2705 break;
2706 case 3:
2707 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2710 NUM_BANKS(ADDR_SURF_16_BANK));
2711 break;
2712 case 4:
2713 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2714 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2715 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2716 NUM_BANKS(ADDR_SURF_16_BANK));
2717 break;
2718 case 5:
2719 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2720 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2721 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2722 NUM_BANKS(ADDR_SURF_8_BANK));
2723 break;
2724 case 6:
2725 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2728 NUM_BANKS(ADDR_SURF_4_BANK));
2729 break;
2730 case 8:
2731 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2732 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2733 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2734 NUM_BANKS(ADDR_SURF_16_BANK));
2735 break;
2736 case 9:
2737 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2738 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2739 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2740 NUM_BANKS(ADDR_SURF_16_BANK));
2741 break;
2742 case 10:
2743 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2746 NUM_BANKS(ADDR_SURF_16_BANK));
2747 break;
2748 case 11:
2749 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2750 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2751 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2752 NUM_BANKS(ADDR_SURF_16_BANK));
2753 break;
2754 case 12:
2755 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2756 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2757 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2758 NUM_BANKS(ADDR_SURF_16_BANK));
2759 break;
2760 case 13:
2761 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2764 NUM_BANKS(ADDR_SURF_8_BANK));
2765 break;
2766 case 14:
2767 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2768 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2769 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2770 NUM_BANKS(ADDR_SURF_4_BANK));
2771 break;
2772 default:
2773 gb_tile_moden = 0;
2774 break;
2775 }
2776 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2777 }
2778 } else if (num_pipe_configs == 2) {
2779 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2780 switch (reg_offset) {
2781 case 0:
2782 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2783 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2784 PIPE_CONFIG(ADDR_SURF_P2) |
2785 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2786 break;
2787 case 1:
2788 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2789 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2790 PIPE_CONFIG(ADDR_SURF_P2) |
2791 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2792 break;
2793 case 2:
2794 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2795 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2796 PIPE_CONFIG(ADDR_SURF_P2) |
2797 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2798 break;
2799 case 3:
2800 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2801 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2802 PIPE_CONFIG(ADDR_SURF_P2) |
2803 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2804 break;
2805 case 4:
2806 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2807 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2808 PIPE_CONFIG(ADDR_SURF_P2) |
2809 TILE_SPLIT(split_equal_to_row_size));
2810 break;
2811 case 5:
2812 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2813 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2814 break;
2815 case 6:
2816 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2817 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2818 PIPE_CONFIG(ADDR_SURF_P2) |
2819 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2820 break;
2821 case 7:
2822 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2823 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2824 PIPE_CONFIG(ADDR_SURF_P2) |
2825 TILE_SPLIT(split_equal_to_row_size));
2826 break;
2827 case 8:
2828 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2829 break;
2830 case 9:
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2832 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2833 break;
2834 case 10:
2835 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2836 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2837 PIPE_CONFIG(ADDR_SURF_P2) |
2838 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2839 break;
2840 case 11:
2841 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2842 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2843 PIPE_CONFIG(ADDR_SURF_P2) |
2844 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2845 break;
2846 case 12:
2847 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2848 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2849 PIPE_CONFIG(ADDR_SURF_P2) |
2850 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2851 break;
2852 case 13:
2853 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2854 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2855 break;
2856 case 14:
2857 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2858 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2859 PIPE_CONFIG(ADDR_SURF_P2) |
2860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2861 break;
2862 case 16:
2863 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2864 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2865 PIPE_CONFIG(ADDR_SURF_P2) |
2866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2867 break;
2868 case 17:
2869 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2870 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2871 PIPE_CONFIG(ADDR_SURF_P2) |
2872 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2873 break;
2874 case 27:
2875 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2876 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2877 break;
2878 case 28:
2879 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2880 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2881 PIPE_CONFIG(ADDR_SURF_P2) |
2882 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2883 break;
2884 case 29:
2885 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2886 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2887 PIPE_CONFIG(ADDR_SURF_P2) |
2888 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2889 break;
2890 case 30:
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P2) |
2894 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2895 break;
2896 default:
2897 gb_tile_moden = 0;
2898 break;
2899 }
39aee490 2900 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2901 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2902 }
2903 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2904 switch (reg_offset) {
2905 case 0:
2906 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2907 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2908 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2909 NUM_BANKS(ADDR_SURF_16_BANK));
2910 break;
2911 case 1:
2912 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2913 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2914 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2915 NUM_BANKS(ADDR_SURF_16_BANK));
2916 break;
2917 case 2:
2918 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2919 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2920 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2921 NUM_BANKS(ADDR_SURF_16_BANK));
2922 break;
2923 case 3:
2924 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2925 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2926 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2927 NUM_BANKS(ADDR_SURF_16_BANK));
2928 break;
2929 case 4:
2930 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2931 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2932 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2933 NUM_BANKS(ADDR_SURF_16_BANK));
2934 break;
2935 case 5:
2936 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2937 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2938 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2939 NUM_BANKS(ADDR_SURF_16_BANK));
2940 break;
2941 case 6:
2942 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2943 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2944 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2945 NUM_BANKS(ADDR_SURF_8_BANK));
2946 break;
2947 case 8:
2948 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2949 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2950 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2951 NUM_BANKS(ADDR_SURF_16_BANK));
2952 break;
2953 case 9:
2954 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2957 NUM_BANKS(ADDR_SURF_16_BANK));
2958 break;
2959 case 10:
2960 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2961 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2962 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2963 NUM_BANKS(ADDR_SURF_16_BANK));
2964 break;
2965 case 11:
2966 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2967 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2968 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2969 NUM_BANKS(ADDR_SURF_16_BANK));
2970 break;
2971 case 12:
2972 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2975 NUM_BANKS(ADDR_SURF_16_BANK));
2976 break;
2977 case 13:
2978 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2979 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2980 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2981 NUM_BANKS(ADDR_SURF_16_BANK));
2982 break;
2983 case 14:
2984 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2987 NUM_BANKS(ADDR_SURF_8_BANK));
2988 break;
2989 default:
2990 gb_tile_moden = 0;
2991 break;
2992 }
2993 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2994 }
2995 } else
2996 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2997}
2998
2999/**
3000 * cik_select_se_sh - select which SE, SH to address
3001 *
3002 * @rdev: radeon_device pointer
3003 * @se_num: shader engine to address
3004 * @sh_num: sh block to address
3005 *
3006 * Select which SE, SH combinations to address. Certain
3007 * registers are instanced per SE or SH. 0xffffffff means
3008 * broadcast to all SEs or SHs (CIK).
3009 */
3010static void cik_select_se_sh(struct radeon_device *rdev,
3011 u32 se_num, u32 sh_num)
3012{
3013 u32 data = INSTANCE_BROADCAST_WRITES;
3014
3015 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3016 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3017 else if (se_num == 0xffffffff)
3018 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3019 else if (sh_num == 0xffffffff)
3020 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3021 else
3022 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3023 WREG32(GRBM_GFX_INDEX, data);
3024}
3025
3026/**
3027 * cik_create_bitmask - create a bitmask
3028 *
3029 * @bit_width: length of the mask
3030 *
3031 * create a variable length bit mask (CIK).
3032 * Returns the bitmask.
3033 */
3034static u32 cik_create_bitmask(u32 bit_width)
3035{
3036 u32 i, mask = 0;
3037
3038 for (i = 0; i < bit_width; i++) {
3039 mask <<= 1;
3040 mask |= 1;
3041 }
3042 return mask;
3043}
3044
3045/**
3046 * cik_select_se_sh - select which SE, SH to address
3047 *
3048 * @rdev: radeon_device pointer
3049 * @max_rb_num: max RBs (render backends) for the asic
3050 * @se_num: number of SEs (shader engines) for the asic
3051 * @sh_per_se: number of SH blocks per SE for the asic
3052 *
3053 * Calculates the bitmask of disabled RBs (CIK).
3054 * Returns the disabled RB bitmask.
3055 */
3056static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3057 u32 max_rb_num, u32 se_num,
3058 u32 sh_per_se)
3059{
3060 u32 data, mask;
3061
3062 data = RREG32(CC_RB_BACKEND_DISABLE);
3063 if (data & 1)
3064 data &= BACKEND_DISABLE_MASK;
3065 else
3066 data = 0;
3067 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3068
3069 data >>= BACKEND_DISABLE_SHIFT;
3070
3071 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
3072
3073 return data & mask;
3074}
3075
3076/**
3077 * cik_setup_rb - setup the RBs on the asic
3078 *
3079 * @rdev: radeon_device pointer
3080 * @se_num: number of SEs (shader engines) for the asic
3081 * @sh_per_se: number of SH blocks per SE for the asic
3082 * @max_rb_num: max RBs (render backends) for the asic
3083 *
3084 * Configures per-SE/SH RB registers (CIK).
3085 */
3086static void cik_setup_rb(struct radeon_device *rdev,
3087 u32 se_num, u32 sh_per_se,
3088 u32 max_rb_num)
3089{
3090 int i, j;
3091 u32 data, mask;
3092 u32 disabled_rbs = 0;
3093 u32 enabled_rbs = 0;
3094
3095 for (i = 0; i < se_num; i++) {
3096 for (j = 0; j < sh_per_se; j++) {
3097 cik_select_se_sh(rdev, i, j);
3098 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
fc821b70
AD		 3099 if (rdev->family == CHIP_HAWAII)
3100 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3101 else
3102 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3103 }
3104 }
3105 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3106
3107 mask = 1;
3108 for (i = 0; i < max_rb_num; i++) {
3109 if (!(disabled_rbs & mask))
3110 enabled_rbs |= mask;
3111 mask <<= 1;
3112 }
3113
3114 for (i = 0; i < se_num; i++) {
3115 cik_select_se_sh(rdev, i, 0xffffffff);
3116 data = 0;
3117 for (j = 0; j < sh_per_se; j++) {
3118 switch (enabled_rbs & 3) {
fc821b70
AD		 3119 case 0:
3120 if (j == 0)
3121 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3122 else
3123 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3124 break;
8cc1a532
AD		 3125 case 1:
3126 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3127 break;
3128 case 2:
3129 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3130 break;
3131 case 3:
3132 default:
3133 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3134 break;
3135 }
3136 enabled_rbs >>= 2;
3137 }
3138 WREG32(PA_SC_RASTER_CONFIG, data);
3139 }
3140 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3141}
3142
3143/**
3144 * cik_gpu_init - setup the 3D engine
3145 *
3146 * @rdev: radeon_device pointer
3147 *
3148 * Configures the 3D engine and tiling configuration
3149 * registers so that the 3D engine is usable.
3150 */
3151static void cik_gpu_init(struct radeon_device *rdev)
3152{
3153 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3154 u32 mc_shared_chmap, mc_arb_ramcfg;
3155 u32 hdp_host_path_cntl;
3156 u32 tmp;
3157 int i, j;
3158
3159 switch (rdev->family) {
3160 case CHIP_BONAIRE:
3161 rdev->config.cik.max_shader_engines = 2;
3162 rdev->config.cik.max_tile_pipes = 4;
3163 rdev->config.cik.max_cu_per_sh = 7;
3164 rdev->config.cik.max_sh_per_se = 1;
3165 rdev->config.cik.max_backends_per_se = 2;
3166 rdev->config.cik.max_texture_channel_caches = 4;
3167 rdev->config.cik.max_gprs = 256;
3168 rdev->config.cik.max_gs_threads = 32;
3169 rdev->config.cik.max_hw_contexts = 8;
3170
3171 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3172 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3173 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3174 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3175 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3176 break;
b496038b
AD		 3177 case CHIP_HAWAII:
3178 rdev->config.cik.max_shader_engines = 4;
3179 rdev->config.cik.max_tile_pipes = 16;
3180 rdev->config.cik.max_cu_per_sh = 11;
3181 rdev->config.cik.max_sh_per_se = 1;
3182 rdev->config.cik.max_backends_per_se = 4;
3183 rdev->config.cik.max_texture_channel_caches = 16;
3184 rdev->config.cik.max_gprs = 256;
3185 rdev->config.cik.max_gs_threads = 32;
3186 rdev->config.cik.max_hw_contexts = 8;
3187
3188 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3189 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3190 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3191 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3192 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3193 break;
8cc1a532 3194 case CHIP_KAVERI:
b2e4c70a
AD		 3195 rdev->config.cik.max_shader_engines = 1;
3196 rdev->config.cik.max_tile_pipes = 4;
3197 if ((rdev->pdev->device == 0x1304) ||
3198 (rdev->pdev->device == 0x1305) ||
3199 (rdev->pdev->device == 0x130C) ||
3200 (rdev->pdev->device == 0x130F) ||
3201 (rdev->pdev->device == 0x1310) ||
3202 (rdev->pdev->device == 0x1311) ||
3203 (rdev->pdev->device == 0x131C)) {
3204 rdev->config.cik.max_cu_per_sh = 8;
3205 rdev->config.cik.max_backends_per_se = 2;
3206 } else if ((rdev->pdev->device == 0x1309) ||
3207 (rdev->pdev->device == 0x130A) ||
3208 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 3209 (rdev->pdev->device == 0x1313) ||
3210 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 3211 rdev->config.cik.max_cu_per_sh = 6;
3212 rdev->config.cik.max_backends_per_se = 2;
3213 } else if ((rdev->pdev->device == 0x1306) ||
3214 (rdev->pdev->device == 0x1307) ||
3215 (rdev->pdev->device == 0x130B) ||
3216 (rdev->pdev->device == 0x130E) ||
3217 (rdev->pdev->device == 0x1315) ||
3218 (rdev->pdev->device == 0x131B)) {
3219 rdev->config.cik.max_cu_per_sh = 4;
3220 rdev->config.cik.max_backends_per_se = 1;
3221 } else {
3222 rdev->config.cik.max_cu_per_sh = 3;
3223 rdev->config.cik.max_backends_per_se = 1;
3224 }
3225 rdev->config.cik.max_sh_per_se = 1;
3226 rdev->config.cik.max_texture_channel_caches = 4;
3227 rdev->config.cik.max_gprs = 256;
3228 rdev->config.cik.max_gs_threads = 16;
3229 rdev->config.cik.max_hw_contexts = 8;
3230
3231 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3232 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3233 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3234 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3235 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3236 break;
3237 case CHIP_KABINI:
3238 default:
3239 rdev->config.cik.max_shader_engines = 1;
3240 rdev->config.cik.max_tile_pipes = 2;
3241 rdev->config.cik.max_cu_per_sh = 2;
3242 rdev->config.cik.max_sh_per_se = 1;
3243 rdev->config.cik.max_backends_per_se = 1;
3244 rdev->config.cik.max_texture_channel_caches = 2;
3245 rdev->config.cik.max_gprs = 256;
3246 rdev->config.cik.max_gs_threads = 16;
3247 rdev->config.cik.max_hw_contexts = 8;
3248
3249 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3250 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3251 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3252 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3253 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3254 break;
3255 }
3256
3257 /* Initialize HDP */
3258 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3259 WREG32((0x2c14 + j), 0x00000000);
3260 WREG32((0x2c18 + j), 0x00000000);
3261 WREG32((0x2c1c + j), 0x00000000);
3262 WREG32((0x2c20 + j), 0x00000000);
3263 WREG32((0x2c24 + j), 0x00000000);
3264 }
3265
3266 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3267
3268 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3269
3270 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3271 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3272
3273 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3274 rdev->config.cik.mem_max_burst_length_bytes = 256;
3275 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3276 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3277 if (rdev->config.cik.mem_row_size_in_kb > 4)
3278 rdev->config.cik.mem_row_size_in_kb = 4;
3279 /* XXX use MC settings? */
3280 rdev->config.cik.shader_engine_tile_size = 32;
3281 rdev->config.cik.num_gpus = 1;
3282 rdev->config.cik.multi_gpu_tile_size = 64;
3283
3284 /* fix up row size */
3285 gb_addr_config &= ~ROW_SIZE_MASK;
3286 switch (rdev->config.cik.mem_row_size_in_kb) {
3287 case 1:
3288 default:
3289 gb_addr_config |= ROW_SIZE(0);
3290 break;
3291 case 2:
3292 gb_addr_config |= ROW_SIZE(1);
3293 break;
3294 case 4:
3295 gb_addr_config |= ROW_SIZE(2);
3296 break;
3297 }
3298
3299 /* setup tiling info dword. gb_addr_config is not adequate since it does
3300 * not have bank info, so create a custom tiling dword.
3301 * bits 3:0 num_pipes
3302 * bits 7:4 num_banks
3303 * bits 11:8 group_size
3304 * bits 15:12 row_size
3305 */
3306 rdev->config.cik.tile_config = 0;
3307 switch (rdev->config.cik.num_tile_pipes) {
3308 case 1:
3309 rdev->config.cik.tile_config |= (0 << 0);
3310 break;
3311 case 2:
3312 rdev->config.cik.tile_config |= (1 << 0);
3313 break;
3314 case 4:
3315 rdev->config.cik.tile_config |= (2 << 0);
3316 break;
3317 case 8:
3318 default:
3319 /* XXX what about 12? */
3320 rdev->config.cik.tile_config |= (3 << 0);
3321 break;
3322 }
a537314e
MD		 3323 rdev->config.cik.tile_config |=
3324 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3325 rdev->config.cik.tile_config |=
3326 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3327 rdev->config.cik.tile_config |=
3328 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3329
3330 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3331 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3332 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3333 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3334 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3335 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3336 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3337 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3338
3339 cik_tiling_mode_table_init(rdev);
3340
3341 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3342 rdev->config.cik.max_sh_per_se,
3343 rdev->config.cik.max_backends_per_se);
3344
3345 /* set HW defaults for 3D engine */
3346 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3347
3348 WREG32(SX_DEBUG_1, 0x20);
3349
3350 WREG32(TA_CNTL_AUX, 0x00010000);
3351
3352 tmp = RREG32(SPI_CONFIG_CNTL);
3353 tmp |= 0x03000000;
3354 WREG32(SPI_CONFIG_CNTL, tmp);
3355
3356 WREG32(SQ_CONFIG, 1);
3357
3358 WREG32(DB_DEBUG, 0);
3359
3360 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3361 tmp |= 0x00000400;
3362 WREG32(DB_DEBUG2, tmp);
3363
3364 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3365 tmp |= 0x00020200;
3366 WREG32(DB_DEBUG3, tmp);
3367
3368 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3369 tmp |= 0x00018208;
3370 WREG32(CB_HW_CONTROL, tmp);
3371
3372 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3373
3374 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3375 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3376 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3377 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3378
3379 WREG32(VGT_NUM_INSTANCES, 1);
3380
3381 WREG32(CP_PERFMON_CNTL, 0);
3382
3383 WREG32(SQ_CONFIG, 0);
3384
3385 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3386 FORCE_EOV_MAX_REZ_CNT(255)));
3387
3388 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3389 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3390
3391 WREG32(VGT_GS_VERTEX_REUSE, 16);
3392 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3393
3394 tmp = RREG32(HDP_MISC_CNTL);
3395 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3396 WREG32(HDP_MISC_CNTL, tmp);
3397
3398 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3399 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3400
3401 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3402 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3403
3404 udelay(50);
3405}
3406
2cae3bc3
AD		 3407/*
3408 * GPU scratch registers helpers function.
3409 */
3410/**
3411 * cik_scratch_init - setup driver info for CP scratch regs
3412 *
3413 * @rdev: radeon_device pointer
3414 *
3415 * Set up the number and offset of the CP scratch registers.
3416 * NOTE: use of CP scratch registers is a legacy inferface and
3417 * is not used by default on newer asics (r6xx+). On newer asics,
3418 * memory buffers are used for fences rather than scratch regs.
3419 */
3420static void cik_scratch_init(struct radeon_device *rdev)
3421{
3422 int i;
3423
3424 rdev->scratch.num_reg = 7;
3425 rdev->scratch.reg_base = SCRATCH_REG0;
3426 for (i = 0; i < rdev->scratch.num_reg; i++) {
3427 rdev->scratch.free[i] = true;
3428 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3429 }
3430}
3431
fbc832c7
AD		 3432/**
3433 * cik_ring_test - basic gfx ring test
3434 *
3435 * @rdev: radeon_device pointer
3436 * @ring: radeon_ring structure holding ring information
3437 *
3438 * Allocate a scratch register and write to it using the gfx ring (CIK).
3439 * Provides a basic gfx ring test to verify that the ring is working.
3440 * Used by cik_cp_gfx_resume();
3441 * Returns 0 on success, error on failure.
3442 */
3443int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3444{
3445 uint32_t scratch;
3446 uint32_t tmp = 0;
3447 unsigned i;
3448 int r;
3449
3450 r = radeon_scratch_get(rdev, &scratch);
3451 if (r) {
3452 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3453 return r;
3454 }
3455 WREG32(scratch, 0xCAFEDEAD);
3456 r = radeon_ring_lock(rdev, ring, 3);
3457 if (r) {
3458 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3459 radeon_scratch_free(rdev, scratch);
3460 return r;
3461 }
3462 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3463 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3464 radeon_ring_write(ring, 0xDEADBEEF);
3465 radeon_ring_unlock_commit(rdev, ring);
963e81f9 3466
fbc832c7
AD		 3467 for (i = 0; i < rdev->usec_timeout; i++) {
3468 tmp = RREG32(scratch);
3469 if (tmp == 0xDEADBEEF)
3470 break;
3471 DRM_UDELAY(1);
3472 }
3473 if (i < rdev->usec_timeout) {
3474 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3475 } else {
3476 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3477 ring->idx, scratch, tmp);
3478 r = -EINVAL;
3479 }
3480 radeon_scratch_free(rdev, scratch);
3481 return r;
3482}
3483
2cae3bc3 3484/**
b07fdd38 3485 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3486 *
3487 * @rdev: radeon_device pointer
3488 * @fence: radeon fence object
3489 *
3490 * Emits a fence sequnce number on the gfx ring and flushes
3491 * GPU caches.
3492 */
b07fdd38
AD		 3493void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3494 struct radeon_fence *fence)
2cae3bc3
AD		 3495{
3496 struct radeon_ring *ring = &rdev->ring[fence->ring];
3497 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3498
3499 /* EVENT_WRITE_EOP - flush caches, send int */
3500 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3501 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3502 EOP_TC_ACTION_EN |
3503 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3504 EVENT_INDEX(5)));
3505 radeon_ring_write(ring, addr & 0xfffffffc);
3506 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3507 radeon_ring_write(ring, fence->seq);
3508 radeon_ring_write(ring, 0);
3509 /* HDP flush */
3510 /* We should be using the new WAIT_REG_MEM special op packet here
3511 * but it causes the CP to hang
3512 */
3513 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3514 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3515 WRITE_DATA_DST_SEL(0)));
3516 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3517 radeon_ring_write(ring, 0);
3518 radeon_ring_write(ring, 0);
3519}
3520
b07fdd38
AD		 3521/**
3522 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3523 *
3524 * @rdev: radeon_device pointer
3525 * @fence: radeon fence object
3526 *
3527 * Emits a fence sequnce number on the compute ring and flushes
3528 * GPU caches.
3529 */
3530void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3531 struct radeon_fence *fence)
3532{
3533 struct radeon_ring *ring = &rdev->ring[fence->ring];
3534 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3535
3536 /* RELEASE_MEM - flush caches, send int */
3537 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3538 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3539 EOP_TC_ACTION_EN |
3540 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3541 EVENT_INDEX(5)));
3542 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3543 radeon_ring_write(ring, addr & 0xfffffffc);
3544 radeon_ring_write(ring, upper_32_bits(addr));
3545 radeon_ring_write(ring, fence->seq);
3546 radeon_ring_write(ring, 0);
3547 /* HDP flush */
3548 /* We should be using the new WAIT_REG_MEM special op packet here
3549 * but it causes the CP to hang
3550 */
3551 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3552 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3553 WRITE_DATA_DST_SEL(0)));
3554 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3555 radeon_ring_write(ring, 0);
3556 radeon_ring_write(ring, 0);
3557}
3558
1654b817 3559bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 3560 struct radeon_ring *ring,
3561 struct radeon_semaphore *semaphore,
3562 bool emit_wait)
3563{
3564 uint64_t addr = semaphore->gpu_addr;
3565 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3566
3567 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3568 radeon_ring_write(ring, addr & 0xffffffff);
3569 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817
CK		 3570
3571 return true;
2cae3bc3
AD		 3572}
3573
c9dbd705
AD		 3574/**
3575 * cik_copy_cpdma - copy pages using the CP DMA engine
3576 *
3577 * @rdev: radeon_device pointer
3578 * @src_offset: src GPU address
3579 * @dst_offset: dst GPU address
3580 * @num_gpu_pages: number of GPU pages to xfer
3581 * @fence: radeon fence object
3582 *
3583 * Copy GPU paging using the CP DMA engine (CIK+).
3584 * Used by the radeon ttm implementation to move pages if
3585 * registered as the asic copy callback.
3586 */
3587int cik_copy_cpdma(struct radeon_device *rdev,
3588 uint64_t src_offset, uint64_t dst_offset,
3589 unsigned num_gpu_pages,
3590 struct radeon_fence **fence)
3591{
3592 struct radeon_semaphore *sem = NULL;
3593 int ring_index = rdev->asic->copy.blit_ring_index;
3594 struct radeon_ring *ring = &rdev->ring[ring_index];
3595 u32 size_in_bytes, cur_size_in_bytes, control;
3596 int i, num_loops;
3597 int r = 0;
3598
3599 r = radeon_semaphore_create(rdev, &sem);
3600 if (r) {
3601 DRM_ERROR("radeon: moving bo (%d).\n", r);
3602 return r;
3603 }
3604
3605 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3606 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3607 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3608 if (r) {
3609 DRM_ERROR("radeon: moving bo (%d).\n", r);
3610 radeon_semaphore_free(rdev, &sem, NULL);
3611 return r;
3612 }
3613
1654b817
CK		 3614 radeon_semaphore_sync_to(sem, *fence);
3615 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
c9dbd705
AD		 3616
3617 for (i = 0; i < num_loops; i++) {
3618 cur_size_in_bytes = size_in_bytes;
3619 if (cur_size_in_bytes > 0x1fffff)
3620 cur_size_in_bytes = 0x1fffff;
3621 size_in_bytes -= cur_size_in_bytes;
3622 control = 0;
3623 if (size_in_bytes == 0)
3624 control |= PACKET3_DMA_DATA_CP_SYNC;
3625 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3626 radeon_ring_write(ring, control);
3627 radeon_ring_write(ring, lower_32_bits(src_offset));
3628 radeon_ring_write(ring, upper_32_bits(src_offset));
3629 radeon_ring_write(ring, lower_32_bits(dst_offset));
3630 radeon_ring_write(ring, upper_32_bits(dst_offset));
3631 radeon_ring_write(ring, cur_size_in_bytes);
3632 src_offset += cur_size_in_bytes;
3633 dst_offset += cur_size_in_bytes;
3634 }
3635
3636 r = radeon_fence_emit(rdev, fence, ring->idx);
3637 if (r) {
3638 radeon_ring_unlock_undo(rdev, ring);
3639 return r;
3640 }
3641
3642 radeon_ring_unlock_commit(rdev, ring);
3643 radeon_semaphore_free(rdev, &sem, *fence);
3644
3645 return r;
3646}
3647
2cae3bc3
AD		 3648/*
3649 * IB stuff
3650 */
3651/**
3652 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3653 *
3654 * @rdev: radeon_device pointer
3655 * @ib: radeon indirect buffer object
3656 *
3657 * Emits an DE (drawing engine) or CE (constant engine) IB
3658 * on the gfx ring. IBs are usually generated by userspace
3659 * acceleration drivers and submitted to the kernel for
3660 * sheduling on the ring. This function schedules the IB
3661 * on the gfx ring for execution by the GPU.
3662 */
3663void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3664{
3665 struct radeon_ring *ring = &rdev->ring[ib->ring];
3666 u32 header, control = INDIRECT_BUFFER_VALID;
3667
3668 if (ib->is_const_ib) {
3669 /* set switch buffer packet before const IB */
3670 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3671 radeon_ring_write(ring, 0);
3672
3673 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3674 } else {
3675 u32 next_rptr;
3676 if (ring->rptr_save_reg) {
3677 next_rptr = ring->wptr + 3 + 4;
3678 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3679 radeon_ring_write(ring, ((ring->rptr_save_reg -
3680 PACKET3_SET_UCONFIG_REG_START) >> 2));
3681 radeon_ring_write(ring, next_rptr);
3682 } else if (rdev->wb.enabled) {
3683 next_rptr = ring->wptr + 5 + 4;
3684 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3685 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3686 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3687 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3688 radeon_ring_write(ring, next_rptr);
3689 }
3690
3691 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3692 }
3693
3694 control |= ib->length_dw |
3695 (ib->vm ? (ib->vm->id << 24) : 0);
3696
3697 radeon_ring_write(ring, header);
3698 radeon_ring_write(ring,
3699#ifdef __BIG_ENDIAN
3700 (2 << 0) |
3701#endif
3702 (ib->gpu_addr & 0xFFFFFFFC));
3703 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3704 radeon_ring_write(ring, control);
3705}
3706
fbc832c7
AD		 3707/**
3708 * cik_ib_test - basic gfx ring IB test
3709 *
3710 * @rdev: radeon_device pointer
3711 * @ring: radeon_ring structure holding ring information
3712 *
3713 * Allocate an IB and execute it on the gfx ring (CIK).
3714 * Provides a basic gfx ring test to verify that IBs are working.
3715 * Returns 0 on success, error on failure.
3716 */
3717int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3718{
3719 struct radeon_ib ib;
3720 uint32_t scratch;
3721 uint32_t tmp = 0;
3722 unsigned i;
3723 int r;
3724
3725 r = radeon_scratch_get(rdev, &scratch);
3726 if (r) {
3727 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3728 return r;
3729 }
3730 WREG32(scratch, 0xCAFEDEAD);
3731 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3732 if (r) {
3733 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 3734 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 3735 return r;
3736 }
3737 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3738 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3739 ib.ptr[2] = 0xDEADBEEF;
3740 ib.length_dw = 3;
3741 r = radeon_ib_schedule(rdev, &ib, NULL);
3742 if (r) {
3743 radeon_scratch_free(rdev, scratch);
3744 radeon_ib_free(rdev, &ib);
3745 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3746 return r;
3747 }
3748 r = radeon_fence_wait(ib.fence, false);
3749 if (r) {
3750 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 3751 radeon_scratch_free(rdev, scratch);
3752 radeon_ib_free(rdev, &ib);
fbc832c7
AD		 3753 return r;
3754 }
3755 for (i = 0; i < rdev->usec_timeout; i++) {
3756 tmp = RREG32(scratch);
3757 if (tmp == 0xDEADBEEF)
3758 break;
3759 DRM_UDELAY(1);
3760 }
3761 if (i < rdev->usec_timeout) {
3762 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3763 } else {
3764 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3765 scratch, tmp);
3766 r = -EINVAL;
3767 }
3768 radeon_scratch_free(rdev, scratch);
3769 radeon_ib_free(rdev, &ib);
3770 return r;
3771}
3772
841cf442
AD		 3773/*
3774 * CP.
3775 * On CIK, gfx and compute now have independant command processors.
3776 *
3777 * GFX
3778 * Gfx consists of a single ring and can process both gfx jobs and
3779 * compute jobs. The gfx CP consists of three microengines (ME):
3780 * PFP - Pre-Fetch Parser
3781 * ME - Micro Engine
3782 * CE - Constant Engine
3783 * The PFP and ME make up what is considered the Drawing Engine (DE).
3784 * The CE is an asynchronous engine used for updating buffer desciptors
3785 * used by the DE so that they can be loaded into cache in parallel
3786 * while the DE is processing state update packets.
3787 *
3788 * Compute
3789 * The compute CP consists of two microengines (ME):
3790 * MEC1 - Compute MicroEngine 1
3791 * MEC2 - Compute MicroEngine 2
3792 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3793 * The queues are exposed to userspace and are programmed directly
3794 * by the compute runtime.
3795 */
3796/**
3797 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3798 *
3799 * @rdev: radeon_device pointer
3800 * @enable: enable or disable the MEs
3801 *
3802 * Halts or unhalts the gfx MEs.
3803 */
3804static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3805{
3806 if (enable)
3807 WREG32(CP_ME_CNTL, 0);
3808 else {
3809 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3810 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3811 }
3812 udelay(50);
3813}
3814
3815/**
3816 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3817 *
3818 * @rdev: radeon_device pointer
3819 *
3820 * Loads the gfx PFP, ME, and CE ucode.
3821 * Returns 0 for success, -EINVAL if the ucode is not available.
3822 */
3823static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3824{
3825 const __be32 *fw_data;
3826 int i;
3827
3828 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3829 return -EINVAL;
3830
3831 cik_cp_gfx_enable(rdev, false);
3832
3833 /* PFP */
3834 fw_data = (const __be32 *)rdev->pfp_fw->data;
3835 WREG32(CP_PFP_UCODE_ADDR, 0);
3836 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3837 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3838 WREG32(CP_PFP_UCODE_ADDR, 0);
3839
3840 /* CE */
3841 fw_data = (const __be32 *)rdev->ce_fw->data;
3842 WREG32(CP_CE_UCODE_ADDR, 0);
3843 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3844 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3845 WREG32(CP_CE_UCODE_ADDR, 0);
3846
3847 /* ME */
3848 fw_data = (const __be32 *)rdev->me_fw->data;
3849 WREG32(CP_ME_RAM_WADDR, 0);
3850 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3851 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3852 WREG32(CP_ME_RAM_WADDR, 0);
3853
3854 WREG32(CP_PFP_UCODE_ADDR, 0);
3855 WREG32(CP_CE_UCODE_ADDR, 0);
3856 WREG32(CP_ME_RAM_WADDR, 0);
3857 WREG32(CP_ME_RAM_RADDR, 0);
3858 return 0;
3859}
3860
3861/**
3862 * cik_cp_gfx_start - start the gfx ring
3863 *
3864 * @rdev: radeon_device pointer
3865 *
3866 * Enables the ring and loads the clear state context and other
3867 * packets required to init the ring.
3868 * Returns 0 for success, error for failure.
3869 */
3870static int cik_cp_gfx_start(struct radeon_device *rdev)
3871{
3872 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3873 int r, i;
3874
3875 /* init the CP */
3876 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3877 WREG32(CP_ENDIAN_SWAP, 0);
3878 WREG32(CP_DEVICE_ID, 1);
3879
3880 cik_cp_gfx_enable(rdev, true);
3881
3882 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3883 if (r) {
3884 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3885 return r;
3886 }
3887
3888 /* init the CE partitions. CE only used for gfx on CIK */
3889 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3890 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3891 radeon_ring_write(ring, 0xc000);
3892 radeon_ring_write(ring, 0xc000);
3893
3894 /* setup clear context state */
3895 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3896 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3897
3898 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3899 radeon_ring_write(ring, 0x80000000);
3900 radeon_ring_write(ring, 0x80000000);
3901
3902 for (i = 0; i < cik_default_size; i++)
3903 radeon_ring_write(ring, cik_default_state[i]);
3904
3905 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3906 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3907
3908 /* set clear context state */
3909 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3910 radeon_ring_write(ring, 0);
3911
3912 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3913 radeon_ring_write(ring, 0x00000316);
3914 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3915 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3916
3917 radeon_ring_unlock_commit(rdev, ring);
3918
3919 return 0;
3920}
3921
3922/**
3923 * cik_cp_gfx_fini - stop the gfx ring
3924 *
3925 * @rdev: radeon_device pointer
3926 *
3927 * Stop the gfx ring and tear down the driver ring
3928 * info.
3929 */
3930static void cik_cp_gfx_fini(struct radeon_device *rdev)
3931{
3932 cik_cp_gfx_enable(rdev, false);
3933 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3934}
3935
3936/**
3937 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3938 *
3939 * @rdev: radeon_device pointer
3940 *
3941 * Program the location and size of the gfx ring buffer
3942 * and test it to make sure it's working.
3943 * Returns 0 for success, error for failure.
3944 */
3945static int cik_cp_gfx_resume(struct radeon_device *rdev)
3946{
3947 struct radeon_ring *ring;
3948 u32 tmp;
3949 u32 rb_bufsz;
3950 u64 rb_addr;
3951 int r;
3952
3953 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 3954 if (rdev->family != CHIP_HAWAII)
3955 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 3956
3957 /* Set the write pointer delay */
3958 WREG32(CP_RB_WPTR_DELAY, 0);
3959
3960 /* set the RB to use vmid 0 */
3961 WREG32(CP_RB_VMID, 0);
3962
3963 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3964
3965 /* ring 0 - compute and gfx */
3966 /* Set ring buffer size */
3967 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3968 rb_bufsz = order_base_2(ring->ring_size / 8);
3969 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 3970#ifdef __BIG_ENDIAN
3971 tmp |= BUF_SWAP_32BIT;
3972#endif
3973 WREG32(CP_RB0_CNTL, tmp);
3974
3975 /* Initialize the ring buffer's read and write pointers */
3976 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3977 ring->wptr = 0;
3978 WREG32(CP_RB0_WPTR, ring->wptr);
3979
3980 /* set the wb address wether it's enabled or not */
3981 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3982 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3983
3984 /* scratch register shadowing is no longer supported */
3985 WREG32(SCRATCH_UMSK, 0);
3986
3987 if (!rdev->wb.enabled)
3988 tmp |= RB_NO_UPDATE;
3989
3990 mdelay(1);
3991 WREG32(CP_RB0_CNTL, tmp);
3992
3993 rb_addr = ring->gpu_addr >> 8;
3994 WREG32(CP_RB0_BASE, rb_addr);
3995 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
3996
3997 ring->rptr = RREG32(CP_RB0_RPTR);
3998
3999 /* start the ring */
4000 cik_cp_gfx_start(rdev);
4001 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4002 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4003 if (r) {
4004 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4005 return r;
4006 }
4007 return 0;
4008}
4009
963e81f9
AD		 4010u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
4011 struct radeon_ring *ring)
4012{
4013 u32 rptr;
4014
4015
4016
4017 if (rdev->wb.enabled) {
4018 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
4019 } else {
f61d5b46 4020 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4021 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4022 rptr = RREG32(CP_HQD_PQ_RPTR);
4023 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4024 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4025 }
963e81f9
AD		 4026
4027 return rptr;
4028}
4029
4030u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
4031 struct radeon_ring *ring)
4032{
4033 u32 wptr;
4034
4035 if (rdev->wb.enabled) {
4036 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
4037 } else {
f61d5b46 4038 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4039 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4040 wptr = RREG32(CP_HQD_PQ_WPTR);
4041 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4042 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4043 }
963e81f9
AD		 4044
4045 return wptr;
4046}
4047
4048void cik_compute_ring_set_wptr(struct radeon_device *rdev,
4049 struct radeon_ring *ring)
4050{
2e1e6dad
CK		 4051 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
4052 WDOORBELL32(ring->doorbell_offset, ring->wptr);
963e81f9
AD		 4053}
4054
841cf442
AD		 4055/**
4056 * cik_cp_compute_enable - enable/disable the compute CP MEs
4057 *
4058 * @rdev: radeon_device pointer
4059 * @enable: enable or disable the MEs
4060 *
4061 * Halts or unhalts the compute MEs.
4062 */
4063static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4064{
4065 if (enable)
4066 WREG32(CP_MEC_CNTL, 0);
4067 else
4068 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4069 udelay(50);
4070}
4071
4072/**
4073 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4074 *
4075 * @rdev: radeon_device pointer
4076 *
4077 * Loads the compute MEC1&2 ucode.
4078 * Returns 0 for success, -EINVAL if the ucode is not available.
4079 */
4080static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4081{
4082 const __be32 *fw_data;
4083 int i;
4084
4085 if (!rdev->mec_fw)
4086 return -EINVAL;
4087
4088 cik_cp_compute_enable(rdev, false);
4089
4090 /* MEC1 */
4091 fw_data = (const __be32 *)rdev->mec_fw->data;
4092 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4093 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4094 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4095 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4096
4097 if (rdev->family == CHIP_KAVERI) {
4098 /* MEC2 */
4099 fw_data = (const __be32 *)rdev->mec_fw->data;
4100 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4101 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4102 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4103 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4104 }
4105
4106 return 0;
4107}
4108
4109/**
4110 * cik_cp_compute_start - start the compute queues
4111 *
4112 * @rdev: radeon_device pointer
4113 *
4114 * Enable the compute queues.
4115 * Returns 0 for success, error for failure.
4116 */
4117static int cik_cp_compute_start(struct radeon_device *rdev)
4118{
963e81f9
AD		 4119 cik_cp_compute_enable(rdev, true);
4120
841cf442
AD		 4121 return 0;
4122}
4123
4124/**
4125 * cik_cp_compute_fini - stop the compute queues
4126 *
4127 * @rdev: radeon_device pointer
4128 *
4129 * Stop the compute queues and tear down the driver queue
4130 * info.
4131 */
4132static void cik_cp_compute_fini(struct radeon_device *rdev)
4133{
963e81f9
AD		 4134 int i, idx, r;
4135
841cf442 4136 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4137
4138 for (i = 0; i < 2; i++) {
4139 if (i == 0)
4140 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4141 else
4142 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4143
4144 if (rdev->ring[idx].mqd_obj) {
4145 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4146 if (unlikely(r != 0))
4147 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4148
4149 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4150 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4151
4152 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4153 rdev->ring[idx].mqd_obj = NULL;
4154 }
4155 }
841cf442
AD		 4156}
4157
963e81f9
AD		 4158static void cik_mec_fini(struct radeon_device *rdev)
4159{
4160 int r;
4161
4162 if (rdev->mec.hpd_eop_obj) {
4163 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4164 if (unlikely(r != 0))
4165 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4166 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4167 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4168
4169 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4170 rdev->mec.hpd_eop_obj = NULL;
4171 }
4172}
4173
4174#define MEC_HPD_SIZE 2048
4175
4176static int cik_mec_init(struct radeon_device *rdev)
4177{
4178 int r;
4179 u32 *hpd;
4180
4181 /*
4182 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4183 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4184 */
4185 if (rdev->family == CHIP_KAVERI)
4186 rdev->mec.num_mec = 2;
4187 else
4188 rdev->mec.num_mec = 1;
4189 rdev->mec.num_pipe = 4;
4190 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4191
4192 if (rdev->mec.hpd_eop_obj == NULL) {
4193 r = radeon_bo_create(rdev,
4194 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4195 PAGE_SIZE, true,
4196 RADEON_GEM_DOMAIN_GTT, NULL,
4197 &rdev->mec.hpd_eop_obj);
4198 if (r) {
4199 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4200 return r;
4201 }
4202 }
4203
4204 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4205 if (unlikely(r != 0)) {
4206 cik_mec_fini(rdev);
4207 return r;
4208 }
4209 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4210 &rdev->mec.hpd_eop_gpu_addr);
4211 if (r) {
4212 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4213 cik_mec_fini(rdev);
4214 return r;
4215 }
4216 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4217 if (r) {
4218 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4219 cik_mec_fini(rdev);
4220 return r;
4221 }
4222
4223 /* clear memory. Not sure if this is required or not */
4224 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4225
4226 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4227 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4228
4229 return 0;
4230}
4231
4232struct hqd_registers
4233{
4234 u32 cp_mqd_base_addr;
4235 u32 cp_mqd_base_addr_hi;
4236 u32 cp_hqd_active;
4237 u32 cp_hqd_vmid;
4238 u32 cp_hqd_persistent_state;
4239 u32 cp_hqd_pipe_priority;
4240 u32 cp_hqd_queue_priority;
4241 u32 cp_hqd_quantum;
4242 u32 cp_hqd_pq_base;
4243 u32 cp_hqd_pq_base_hi;
4244 u32 cp_hqd_pq_rptr;
4245 u32 cp_hqd_pq_rptr_report_addr;
4246 u32 cp_hqd_pq_rptr_report_addr_hi;
4247 u32 cp_hqd_pq_wptr_poll_addr;
4248 u32 cp_hqd_pq_wptr_poll_addr_hi;
4249 u32 cp_hqd_pq_doorbell_control;
4250 u32 cp_hqd_pq_wptr;
4251 u32 cp_hqd_pq_control;
4252 u32 cp_hqd_ib_base_addr;
4253 u32 cp_hqd_ib_base_addr_hi;
4254 u32 cp_hqd_ib_rptr;
4255 u32 cp_hqd_ib_control;
4256 u32 cp_hqd_iq_timer;
4257 u32 cp_hqd_iq_rptr;
4258 u32 cp_hqd_dequeue_request;
4259 u32 cp_hqd_dma_offload;
4260 u32 cp_hqd_sema_cmd;
4261 u32 cp_hqd_msg_type;
4262 u32 cp_hqd_atomic0_preop_lo;
4263 u32 cp_hqd_atomic0_preop_hi;
4264 u32 cp_hqd_atomic1_preop_lo;
4265 u32 cp_hqd_atomic1_preop_hi;
4266 u32 cp_hqd_hq_scheduler0;
4267 u32 cp_hqd_hq_scheduler1;
4268 u32 cp_mqd_control;
4269};
4270
4271struct bonaire_mqd
4272{
4273 u32 header;
4274 u32 dispatch_initiator;
4275 u32 dimensions[3];
4276 u32 start_idx[3];
4277 u32 num_threads[3];
4278 u32 pipeline_stat_enable;
4279 u32 perf_counter_enable;
4280 u32 pgm[2];
4281 u32 tba[2];
4282 u32 tma[2];
4283 u32 pgm_rsrc[2];
4284 u32 vmid;
4285 u32 resource_limits;
4286 u32 static_thread_mgmt01[2];
4287 u32 tmp_ring_size;
4288 u32 static_thread_mgmt23[2];
4289 u32 restart[3];
4290 u32 thread_trace_enable;
4291 u32 reserved1;
4292 u32 user_data[16];
4293 u32 vgtcs_invoke_count[2];
4294 struct hqd_registers queue_state;
4295 u32 dequeue_cntr;
4296 u32 interrupt_queue[64];
4297};
4298
841cf442
AD		 4299/**
4300 * cik_cp_compute_resume - setup the compute queue registers
4301 *
4302 * @rdev: radeon_device pointer
4303 *
4304 * Program the compute queues and test them to make sure they
4305 * are working.
4306 * Returns 0 for success, error for failure.
4307 */
4308static int cik_cp_compute_resume(struct radeon_device *rdev)
4309{
963e81f9
AD		 4310 int r, i, idx;
4311 u32 tmp;
4312 bool use_doorbell = true;
4313 u64 hqd_gpu_addr;
4314 u64 mqd_gpu_addr;
4315 u64 eop_gpu_addr;
4316 u64 wb_gpu_addr;
4317 u32 *buf;
4318 struct bonaire_mqd *mqd;
841cf442 4319
841cf442
AD		 4320 r = cik_cp_compute_start(rdev);
4321 if (r)
4322 return r;
963e81f9
AD		 4323
4324 /* fix up chicken bits */
4325 tmp = RREG32(CP_CPF_DEBUG);
4326 tmp |= (1 << 23);
4327 WREG32(CP_CPF_DEBUG, tmp);
4328
4329 /* init the pipes */
f61d5b46 4330 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4331 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4332 int me = (i < 4) ? 1 : 2;
4333 int pipe = (i < 4) ? i : (i - 4);
4334
4335 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4336
4337 cik_srbm_select(rdev, me, pipe, 0, 0);
4338
4339 /* write the EOP addr */
4340 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4341 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4342
4343 /* set the VMID assigned */
4344 WREG32(CP_HPD_EOP_VMID, 0);
4345
4346 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4347 tmp = RREG32(CP_HPD_EOP_CONTROL);
4348 tmp &= ~EOP_SIZE_MASK;
b72a8925 4349 tmp |= order_base_2(MEC_HPD_SIZE / 8);
963e81f9
AD		 4350 WREG32(CP_HPD_EOP_CONTROL, tmp);
4351 }
4352 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4353 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4354
4355 /* init the queues. Just two for now. */
4356 for (i = 0; i < 2; i++) {
4357 if (i == 0)
4358 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4359 else
4360 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4361
4362 if (rdev->ring[idx].mqd_obj == NULL) {
4363 r = radeon_bo_create(rdev,
4364 sizeof(struct bonaire_mqd),
4365 PAGE_SIZE, true,
4366 RADEON_GEM_DOMAIN_GTT, NULL,
4367 &rdev->ring[idx].mqd_obj);
4368 if (r) {
4369 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4370 return r;
4371 }
4372 }
4373
4374 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4375 if (unlikely(r != 0)) {
4376 cik_cp_compute_fini(rdev);
4377 return r;
4378 }
4379 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4380 &mqd_gpu_addr);
4381 if (r) {
4382 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4383 cik_cp_compute_fini(rdev);
4384 return r;
4385 }
4386 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4387 if (r) {
4388 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4389 cik_cp_compute_fini(rdev);
4390 return r;
4391 }
4392
4393 /* doorbell offset */
4394 rdev->ring[idx].doorbell_offset =
4395 (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
4396
4397 /* init the mqd struct */
4398 memset(buf, 0, sizeof(struct bonaire_mqd));
4399
4400 mqd = (struct bonaire_mqd *)buf;
4401 mqd->header = 0xC0310800;
4402 mqd->static_thread_mgmt01[0] = 0xffffffff;
4403 mqd->static_thread_mgmt01[1] = 0xffffffff;
4404 mqd->static_thread_mgmt23[0] = 0xffffffff;
4405 mqd->static_thread_mgmt23[1] = 0xffffffff;
4406
f61d5b46 4407 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4408 cik_srbm_select(rdev, rdev->ring[idx].me,
4409 rdev->ring[idx].pipe,
4410 rdev->ring[idx].queue, 0);
4411
4412 /* disable wptr polling */
4413 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4414 tmp &= ~WPTR_POLL_EN;
4415 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4416
4417 /* enable doorbell? */
4418 mqd->queue_state.cp_hqd_pq_doorbell_control =
4419 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4420 if (use_doorbell)
4421 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4422 else
4423 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4424 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4425 mqd->queue_state.cp_hqd_pq_doorbell_control);
4426
4427 /* disable the queue if it's active */
4428 mqd->queue_state.cp_hqd_dequeue_request = 0;
4429 mqd->queue_state.cp_hqd_pq_rptr = 0;
4430 mqd->queue_state.cp_hqd_pq_wptr= 0;
4431 if (RREG32(CP_HQD_ACTIVE) & 1) {
4432 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4433 for (i = 0; i < rdev->usec_timeout; i++) {
4434 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4435 break;
4436 udelay(1);
4437 }
4438 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4439 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4440 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4441 }
4442
4443 /* set the pointer to the MQD */
4444 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4445 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4446 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4447 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4448 /* set MQD vmid to 0 */
4449 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4450 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4451 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4452
4453 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4454 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4455 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4456 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4457 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4458 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4459
4460 /* set up the HQD, this is similar to CP_RB0_CNTL */
4461 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4462 mqd->queue_state.cp_hqd_pq_control &=
4463 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4464
4465 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4466 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 4467 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4468 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 4469#ifdef __BIG_ENDIAN
4470 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4471#endif
4472 mqd->queue_state.cp_hqd_pq_control &=
4473 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4474 mqd->queue_state.cp_hqd_pq_control |=
4475 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4476 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4477
4478 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4479 if (i == 0)
4480 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4481 else
4482 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4483 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4484 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4485 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4486 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4487 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4488
4489 /* set the wb address wether it's enabled or not */
4490 if (i == 0)
4491 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4492 else
4493 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4494 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4495 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4496 upper_32_bits(wb_gpu_addr) & 0xffff;
4497 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4498 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4499 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4500 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4501
4502 /* enable the doorbell if requested */
4503 if (use_doorbell) {
4504 mqd->queue_state.cp_hqd_pq_doorbell_control =
4505 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4506 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4507 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4508 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
4509 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4510 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4511 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4512
4513 } else {
4514 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4515 }
4516 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4517 mqd->queue_state.cp_hqd_pq_doorbell_control);
4518
4519 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4520 rdev->ring[idx].wptr = 0;
4521 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4522 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4523 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4524 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4525
4526 /* set the vmid for the queue */
4527 mqd->queue_state.cp_hqd_vmid = 0;
4528 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4529
4530 /* activate the queue */
4531 mqd->queue_state.cp_hqd_active = 1;
4532 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4533
4534 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4535 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4536
4537 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4538 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4539
4540 rdev->ring[idx].ready = true;
4541 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4542 if (r)
4543 rdev->ring[idx].ready = false;
4544 }
4545
841cf442
AD		 4546 return 0;
4547}
4548
841cf442
AD		 4549static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4550{
4551 cik_cp_gfx_enable(rdev, enable);
4552 cik_cp_compute_enable(rdev, enable);
4553}
4554
841cf442
AD		 4555static int cik_cp_load_microcode(struct radeon_device *rdev)
4556{
4557 int r;
4558
4559 r = cik_cp_gfx_load_microcode(rdev);
4560 if (r)
4561 return r;
4562 r = cik_cp_compute_load_microcode(rdev);
4563 if (r)
4564 return r;
4565
4566 return 0;
4567}
4568
841cf442
AD		 4569static void cik_cp_fini(struct radeon_device *rdev)
4570{
4571 cik_cp_gfx_fini(rdev);
4572 cik_cp_compute_fini(rdev);
4573}
4574
841cf442
AD		 4575static int cik_cp_resume(struct radeon_device *rdev)
4576{
4577 int r;
4578
4214faf6
AD		 4579 cik_enable_gui_idle_interrupt(rdev, false);
4580
841cf442
AD		 4581 r = cik_cp_load_microcode(rdev);
4582 if (r)
4583 return r;
4584
4585 r = cik_cp_gfx_resume(rdev);
4586 if (r)
4587 return r;
4588 r = cik_cp_compute_resume(rdev);
4589 if (r)
4590 return r;
4591
4214faf6
AD		 4592 cik_enable_gui_idle_interrupt(rdev, true);
4593
841cf442
AD		 4594 return 0;
4595}
4596
cc066715 4597static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 4598{
6f2043ce
AD		 4599 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4600 RREG32(GRBM_STATUS));
4601 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4602 RREG32(GRBM_STATUS2));
4603 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4604 RREG32(GRBM_STATUS_SE0));
4605 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4606 RREG32(GRBM_STATUS_SE1));
4607 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4608 RREG32(GRBM_STATUS_SE2));
4609 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4610 RREG32(GRBM_STATUS_SE3));
4611 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4612 RREG32(SRBM_STATUS));
4613 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4614 RREG32(SRBM_STATUS2));
cc066715
AD		 4615 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4616 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4617 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4618 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 4619 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4620 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4621 RREG32(CP_STALLED_STAT1));
4622 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4623 RREG32(CP_STALLED_STAT2));
4624 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4625 RREG32(CP_STALLED_STAT3));
4626 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4627 RREG32(CP_CPF_BUSY_STAT));
4628 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4629 RREG32(CP_CPF_STALLED_STAT1));
4630 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4631 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4632 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4633 RREG32(CP_CPC_STALLED_STAT1));
4634 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 4635}
6f2043ce 4636
21a93e13 4637/**
cc066715 4638 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 4639 *
4640 * @rdev: radeon_device pointer
21a93e13 4641 *
cc066715
AD		 4642 * Check which blocks are busy and return the relevant reset
4643 * mask to be used by cik_gpu_soft_reset().
4644 * Returns a mask of the blocks to be reset.
21a93e13 4645 */
2483b4ea 4646u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 4647{
cc066715
AD		 4648 u32 reset_mask = 0;
4649 u32 tmp;
21a93e13 4650
cc066715
AD		 4651 /* GRBM_STATUS */
4652 tmp = RREG32(GRBM_STATUS);
4653 if (tmp & (PA_BUSY | SC_BUSY |
4654 BCI_BUSY | SX_BUSY |
4655 TA_BUSY | VGT_BUSY |
4656 DB_BUSY | CB_BUSY |
4657 GDS_BUSY | SPI_BUSY |
4658 IA_BUSY | IA_BUSY_NO_DMA))
4659 reset_mask |= RADEON_RESET_GFX;
21a93e13 4660
cc066715
AD		 4661 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4662 reset_mask |= RADEON_RESET_CP;
21a93e13 4663
cc066715
AD		 4664 /* GRBM_STATUS2 */
4665 tmp = RREG32(GRBM_STATUS2);
4666 if (tmp & RLC_BUSY)
4667 reset_mask |= RADEON_RESET_RLC;
21a93e13 4668
cc066715
AD		 4669 /* SDMA0_STATUS_REG */
4670 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4671 if (!(tmp & SDMA_IDLE))
4672 reset_mask |= RADEON_RESET_DMA;
21a93e13 4673
cc066715
AD		 4674 /* SDMA1_STATUS_REG */
4675 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4676 if (!(tmp & SDMA_IDLE))
4677 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4678
cc066715
AD		 4679 /* SRBM_STATUS2 */
4680 tmp = RREG32(SRBM_STATUS2);
4681 if (tmp & SDMA_BUSY)
4682 reset_mask |= RADEON_RESET_DMA;
21a93e13 4683
cc066715
AD		 4684 if (tmp & SDMA1_BUSY)
4685 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4686
cc066715
AD		 4687 /* SRBM_STATUS */
4688 tmp = RREG32(SRBM_STATUS);
21a93e13 4689
cc066715
AD		 4690 if (tmp & IH_BUSY)
4691 reset_mask |= RADEON_RESET_IH;
21a93e13 4692
cc066715
AD		 4693 if (tmp & SEM_BUSY)
4694 reset_mask |= RADEON_RESET_SEM;
21a93e13 4695
cc066715
AD		 4696 if (tmp & GRBM_RQ_PENDING)
4697 reset_mask |= RADEON_RESET_GRBM;
21a93e13 4698
cc066715
AD		 4699 if (tmp & VMC_BUSY)
4700 reset_mask |= RADEON_RESET_VMC;
21a93e13 4701
cc066715
AD		 4702 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4703 MCC_BUSY | MCD_BUSY))
4704 reset_mask |= RADEON_RESET_MC;
21a93e13 4705
cc066715
AD		 4706 if (evergreen_is_display_hung(rdev))
4707 reset_mask |= RADEON_RESET_DISPLAY;
4708
4709 /* Skip MC reset as it's mostly likely not hung, just busy */
4710 if (reset_mask & RADEON_RESET_MC) {
4711 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4712 reset_mask &= ~RADEON_RESET_MC;
21a93e13 4713 }
cc066715
AD		 4714
4715 return reset_mask;
21a93e13
AD		 4716}
4717
4718/**
cc066715 4719 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 4720 *
4721 * @rdev: radeon_device pointer
cc066715 4722 * @reset_mask: mask of which blocks to reset
21a93e13 4723 *
cc066715 4724 * Soft reset the blocks specified in @reset_mask.
21a93e13 4725 */
cc066715 4726static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 4727{
6f2043ce 4728 struct evergreen_mc_save save;
cc066715
AD		 4729 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4730 u32 tmp;
21a93e13 4731
cc066715
AD		 4732 if (reset_mask == 0)
4733 return;
21a93e13 4734
cc066715 4735 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 4736
cc066715
AD		 4737 cik_print_gpu_status_regs(rdev);
4738 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4739 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4740 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4741 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 4742
fb2c7f4d
AD		 4743 /* disable CG/PG */
4744 cik_fini_pg(rdev);
4745 cik_fini_cg(rdev);
4746
cc066715
AD		 4747 /* stop the rlc */
4748 cik_rlc_stop(rdev);
21a93e13 4749
cc066715
AD		 4750 /* Disable GFX parsing/prefetching */
4751 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 4752
cc066715
AD		 4753 /* Disable MEC parsing/prefetching */
4754 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 4755
cc066715
AD		 4756 if (reset_mask & RADEON_RESET_DMA) {
4757 /* sdma0 */
4758 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4759 tmp |= SDMA_HALT;
4760 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4761 }
4762 if (reset_mask & RADEON_RESET_DMA1) {
4763 /* sdma1 */
4764 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4765 tmp |= SDMA_HALT;
4766 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4767 }
21a93e13 4768
6f2043ce 4769 evergreen_mc_stop(rdev, &save);
cc066715 4770 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 4771 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4772 }
21a93e13 4773
cc066715
AD		 4774 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4775 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 4776
cc066715
AD		 4777 if (reset_mask & RADEON_RESET_CP) {
4778 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 4779
cc066715
AD		 4780 srbm_soft_reset |= SOFT_RESET_GRBM;
4781 }
21a93e13 4782
cc066715
AD		 4783 if (reset_mask & RADEON_RESET_DMA)
4784 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 4785
cc066715
AD		 4786 if (reset_mask & RADEON_RESET_DMA1)
4787 srbm_soft_reset |= SOFT_RESET_SDMA1;
4788
4789 if (reset_mask & RADEON_RESET_DISPLAY)
4790 srbm_soft_reset |= SOFT_RESET_DC;
4791
4792 if (reset_mask & RADEON_RESET_RLC)
4793 grbm_soft_reset |= SOFT_RESET_RLC;
4794
4795 if (reset_mask & RADEON_RESET_SEM)
4796 srbm_soft_reset |= SOFT_RESET_SEM;
4797
4798 if (reset_mask & RADEON_RESET_IH)
4799 srbm_soft_reset |= SOFT_RESET_IH;
4800
4801 if (reset_mask & RADEON_RESET_GRBM)
4802 srbm_soft_reset |= SOFT_RESET_GRBM;
4803
4804 if (reset_mask & RADEON_RESET_VMC)
4805 srbm_soft_reset |= SOFT_RESET_VMC;
4806
4807 if (!(rdev->flags & RADEON_IS_IGP)) {
4808 if (reset_mask & RADEON_RESET_MC)
4809 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 4810 }
4811
cc066715
AD		 4812 if (grbm_soft_reset) {
4813 tmp = RREG32(GRBM_SOFT_RESET);
4814 tmp |= grbm_soft_reset;
4815 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4816 WREG32(GRBM_SOFT_RESET, tmp);
4817 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 4818
cc066715 4819 udelay(50);
21a93e13 4820
cc066715
AD		 4821 tmp &= ~grbm_soft_reset;
4822 WREG32(GRBM_SOFT_RESET, tmp);
4823 tmp = RREG32(GRBM_SOFT_RESET);
4824 }
21a93e13 4825
cc066715
AD		 4826 if (srbm_soft_reset) {
4827 tmp = RREG32(SRBM_SOFT_RESET);
4828 tmp |= srbm_soft_reset;
4829 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4830 WREG32(SRBM_SOFT_RESET, tmp);
4831 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 4832
cc066715 4833 udelay(50);
21a93e13 4834
cc066715
AD		 4835 tmp &= ~srbm_soft_reset;
4836 WREG32(SRBM_SOFT_RESET, tmp);
4837 tmp = RREG32(SRBM_SOFT_RESET);
4838 }
21a93e13 4839
6f2043ce
AD		 4840 /* Wait a little for things to settle down */
4841 udelay(50);
21a93e13 4842
6f2043ce 4843 evergreen_mc_resume(rdev, &save);
cc066715
AD		 4844 udelay(50);
4845
4846 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 4847}
4848
4849/**
cc066715 4850 * cik_asic_reset - soft reset GPU
21a93e13
AD		 4851 *
4852 * @rdev: radeon_device pointer
4853 *
cc066715
AD		 4854 * Look up which blocks are hung and attempt
4855 * to reset them.
6f2043ce 4856 * Returns 0 for success.
21a93e13 4857 */
6f2043ce 4858int cik_asic_reset(struct radeon_device *rdev)
21a93e13 4859{
cc066715 4860 u32 reset_mask;
21a93e13 4861
cc066715 4862 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 4863
cc066715
AD		 4864 if (reset_mask)
4865 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 4866
cc066715 4867 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 4868
cc066715
AD		 4869 reset_mask = cik_gpu_check_soft_reset(rdev);
4870
4871 if (!reset_mask)
4872 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 4873
4874 return 0;
4875}
4876
4877/**
cc066715 4878 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 4879 *
4880 * @rdev: radeon_device pointer
cc066715 4881 * @ring: radeon_ring structure holding ring information
21a93e13 4882 *
cc066715
AD		 4883 * Check if the 3D engine is locked up (CIK).
4884 * Returns true if the engine is locked, false if not.
21a93e13 4885 */
cc066715 4886bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 4887{
cc066715 4888 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 4889
cc066715
AD		 4890 if (!(reset_mask & (RADEON_RESET_GFX |
4891 RADEON_RESET_COMPUTE |
4892 RADEON_RESET_CP))) {
4893 radeon_ring_lockup_update(ring);
4894 return false;
21a93e13 4895 }
cc066715
AD		 4896 /* force CP activities */
4897 radeon_ring_force_activity(rdev, ring);
4898 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 4899}
4900
1c49165d 4901/* MC */
21a93e13 4902/**
1c49165d 4903 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 4904 *
4905 * @rdev: radeon_device pointer
21a93e13 4906 *
1c49165d
AD		 4907 * Set the location of vram, gart, and AGP in the GPU's
4908 * physical address space (CIK).
21a93e13 4909 */
1c49165d 4910static void cik_mc_program(struct radeon_device *rdev)
21a93e13 4911{
1c49165d 4912 struct evergreen_mc_save save;
21a93e13 4913 u32 tmp;
1c49165d 4914 int i, j;
21a93e13 4915
1c49165d
AD		 4916 /* Initialize HDP */
4917 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4918 WREG32((0x2c14 + j), 0x00000000);
4919 WREG32((0x2c18 + j), 0x00000000);
4920 WREG32((0x2c1c + j), 0x00000000);
4921 WREG32((0x2c20 + j), 0x00000000);
4922 WREG32((0x2c24 + j), 0x00000000);
21a93e13 4923 }
1c49165d 4924 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 4925
1c49165d
AD		 4926 evergreen_mc_stop(rdev, &save);
4927 if (radeon_mc_wait_for_idle(rdev)) {
4928 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 4929 }
1c49165d
AD		 4930 /* Lockout access through VGA aperture*/
4931 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4932 /* Update configuration */
4933 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4934 rdev->mc.vram_start >> 12);
4935 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4936 rdev->mc.vram_end >> 12);
4937 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4938 rdev->vram_scratch.gpu_addr >> 12);
4939 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4940 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4941 WREG32(MC_VM_FB_LOCATION, tmp);
4942 /* XXX double check these! */
4943 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4944 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4945 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4946 WREG32(MC_VM_AGP_BASE, 0);
4947 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4948 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4949 if (radeon_mc_wait_for_idle(rdev)) {
4950 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 4951 }
1c49165d
AD		 4952 evergreen_mc_resume(rdev, &save);
4953 /* we need to own VRAM, so turn off the VGA renderer here
4954 * to stop it overwriting our objects */
4955 rv515_vga_render_disable(rdev);
21a93e13
AD		 4956}
4957
4958/**
1c49165d 4959 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 4960 *
4961 * @rdev: radeon_device pointer
21a93e13 4962 *
1c49165d
AD		 4963 * Look up the amount of vram, vram width, and decide how to place
4964 * vram and gart within the GPU's physical address space (CIK).
4965 * Returns 0 for success.
21a93e13 4966 */
1c49165d 4967static int cik_mc_init(struct radeon_device *rdev)
21a93e13 4968{
1c49165d
AD		 4969 u32 tmp;
4970 int chansize, numchan;
21a93e13 4971
1c49165d
AD		 4972 /* Get VRAM informations */
4973 rdev->mc.vram_is_ddr = true;
4974 tmp = RREG32(MC_ARB_RAMCFG);
4975 if (tmp & CHANSIZE_MASK) {
4976 chansize = 64;
21a93e13 4977 } else {
1c49165d 4978 chansize = 32;
21a93e13 4979 }
1c49165d
AD		 4980 tmp = RREG32(MC_SHARED_CHMAP);
4981 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4982 case 0:
4983 default:
4984 numchan = 1;
4985 break;
4986 case 1:
4987 numchan = 2;
4988 break;
4989 case 2:
4990 numchan = 4;
4991 break;
4992 case 3:
4993 numchan = 8;
4994 break;
4995 case 4:
4996 numchan = 3;
4997 break;
4998 case 5:
4999 numchan = 6;
5000 break;
5001 case 6:
5002 numchan = 10;
5003 break;
5004 case 7:
5005 numchan = 12;
5006 break;
5007 case 8:
5008 numchan = 16;
5009 break;
5010 }
5011 rdev->mc.vram_width = numchan * chansize;
5012 /* Could aper size report 0 ? */
5013 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5014 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5015 /* size in MB on si */
13c5bfda
AD		 5016 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5017 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5018 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5019 si_vram_gtt_location(rdev, &rdev->mc);
5020 radeon_update_bandwidth_info(rdev);
5021
5022 return 0;
5023}
5024
5025/*
5026 * GART
5027 * VMID 0 is the physical GPU addresses as used by the kernel.
5028 * VMIDs 1-15 are used for userspace clients and are handled
5029 * by the radeon vm/hsa code.
5030 */
5031/**
5032 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5033 *
5034 * @rdev: radeon_device pointer
5035 *
5036 * Flush the TLB for the VMID 0 page table (CIK).
5037 */
5038void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5039{
5040 /* flush hdp cache */
5041 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5042
5043 /* bits 0-15 are the VM contexts0-15 */
5044 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5045}
5046
5047/**
5048 * cik_pcie_gart_enable - gart enable
5049 *
5050 * @rdev: radeon_device pointer
5051 *
5052 * This sets up the TLBs, programs the page tables for VMID0,
5053 * sets up the hw for VMIDs 1-15 which are allocated on
5054 * demand, and sets up the global locations for the LDS, GDS,
5055 * and GPUVM for FSA64 clients (CIK).
5056 * Returns 0 for success, errors for failure.
5057 */
5058static int cik_pcie_gart_enable(struct radeon_device *rdev)
5059{
5060 int r, i;
5061
5062 if (rdev->gart.robj == NULL) {
5063 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5064 return -EINVAL;
5065 }
5066 r = radeon_gart_table_vram_pin(rdev);
5067 if (r)
5068 return r;
5069 radeon_gart_restore(rdev);
5070 /* Setup TLB control */
5071 WREG32(MC_VM_MX_L1_TLB_CNTL,
5072 (0xA << 7) |
5073 ENABLE_L1_TLB |
5074 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5075 ENABLE_ADVANCED_DRIVER_MODEL |
5076 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5077 /* Setup L2 cache */
5078 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5079 ENABLE_L2_FRAGMENT_PROCESSING |
5080 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5081 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5082 EFFECTIVE_L2_QUEUE_SIZE(7) |
5083 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5084 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5085 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5086 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5087 /* setup context0 */
5088 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5089 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5090 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5091 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5092 (u32)(rdev->dummy_page.addr >> 12));
5093 WREG32(VM_CONTEXT0_CNTL2, 0);
5094 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5095 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5096
5097 WREG32(0x15D4, 0);
5098 WREG32(0x15D8, 0);
5099 WREG32(0x15DC, 0);
5100
5101 /* empty context1-15 */
5102 /* FIXME start with 4G, once using 2 level pt switch to full
5103 * vm size space
5104 */
5105 /* set vm size, must be a multiple of 4 */
5106 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5107 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5108 for (i = 1; i < 16; i++) {
5109 if (i < 8)
5110 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5111 rdev->gart.table_addr >> 12);
5112 else
5113 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5114 rdev->gart.table_addr >> 12);
5115 }
5116
5117 /* enable context1-15 */
5118 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5119 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5120 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5121 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
a00024b0
AD		 5122 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5123 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5124 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5125 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5126 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5127 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5128 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5129 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5130 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5131 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5132 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5133 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d
AD		 5134
5135 /* TC cache setup ??? */
5136 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5137 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5138 WREG32(TC_CFG_L1_STORE_POLICY, 0);
5139
5140 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5141 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5142 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5143 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5144 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5145
5146 WREG32(TC_CFG_L1_VOLATILE, 0);
5147 WREG32(TC_CFG_L2_VOLATILE, 0);
5148
5149 if (rdev->family == CHIP_KAVERI) {
5150 u32 tmp = RREG32(CHUB_CONTROL);
5151 tmp &= ~BYPASS_VM;
5152 WREG32(CHUB_CONTROL, tmp);
5153 }
5154
5155 /* XXX SH_MEM regs */
5156 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5157 mutex_lock(&rdev->srbm_mutex);
1c49165d 5158 for (i = 0; i < 16; i++) {
b556b12e 5159 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5160 /* CP and shaders */
1c49165d
AD		 5161 WREG32(SH_MEM_CONFIG, 0);
5162 WREG32(SH_MEM_APE1_BASE, 1);
5163 WREG32(SH_MEM_APE1_LIMIT, 0);
5164 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5165 /* SDMA GFX */
5166 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5167 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5168 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5169 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5170 /* XXX SDMA RLC - todo */
1c49165d 5171 }
b556b12e 5172 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5173 mutex_unlock(&rdev->srbm_mutex);
1c49165d
AD		 5174
5175 cik_pcie_gart_tlb_flush(rdev);
5176 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5177 (unsigned)(rdev->mc.gtt_size >> 20),
5178 (unsigned long long)rdev->gart.table_addr);
5179 rdev->gart.ready = true;
5180 return 0;
5181}
5182
5183/**
5184 * cik_pcie_gart_disable - gart disable
5185 *
5186 * @rdev: radeon_device pointer
5187 *
5188 * This disables all VM page table (CIK).
5189 */
5190static void cik_pcie_gart_disable(struct radeon_device *rdev)
5191{
5192 /* Disable all tables */
5193 WREG32(VM_CONTEXT0_CNTL, 0);
5194 WREG32(VM_CONTEXT1_CNTL, 0);
5195 /* Setup TLB control */
5196 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5197 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5198 /* Setup L2 cache */
5199 WREG32(VM_L2_CNTL,
5200 ENABLE_L2_FRAGMENT_PROCESSING |
5201 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5202 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5203 EFFECTIVE_L2_QUEUE_SIZE(7) |
5204 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5205 WREG32(VM_L2_CNTL2, 0);
5206 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5207 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5208 radeon_gart_table_vram_unpin(rdev);
5209}
5210
5211/**
5212 * cik_pcie_gart_fini - vm fini callback
5213 *
5214 * @rdev: radeon_device pointer
5215 *
5216 * Tears down the driver GART/VM setup (CIK).
5217 */
5218static void cik_pcie_gart_fini(struct radeon_device *rdev)
5219{
5220 cik_pcie_gart_disable(rdev);
5221 radeon_gart_table_vram_free(rdev);
5222 radeon_gart_fini(rdev);
5223}
5224
5225/* vm parser */
5226/**
5227 * cik_ib_parse - vm ib_parse callback
5228 *
5229 * @rdev: radeon_device pointer
5230 * @ib: indirect buffer pointer
5231 *
5232 * CIK uses hw IB checking so this is a nop (CIK).
5233 */
5234int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5235{
5236 return 0;
5237}
5238
5239/*
5240 * vm
5241 * VMID 0 is the physical GPU addresses as used by the kernel.
5242 * VMIDs 1-15 are used for userspace clients and are handled
5243 * by the radeon vm/hsa code.
5244 */
5245/**
5246 * cik_vm_init - cik vm init callback
5247 *
5248 * @rdev: radeon_device pointer
5249 *
5250 * Inits cik specific vm parameters (number of VMs, base of vram for
5251 * VMIDs 1-15) (CIK).
5252 * Returns 0 for success.
5253 */
5254int cik_vm_init(struct radeon_device *rdev)
5255{
5256 /* number of VMs */
5257 rdev->vm_manager.nvm = 16;
5258 /* base offset of vram pages */
5259 if (rdev->flags & RADEON_IS_IGP) {
5260 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5261 tmp <<= 22;
5262 rdev->vm_manager.vram_base_offset = tmp;
5263 } else
5264 rdev->vm_manager.vram_base_offset = 0;
5265
5266 return 0;
5267}
5268
5269/**
5270 * cik_vm_fini - cik vm fini callback
5271 *
5272 * @rdev: radeon_device pointer
5273 *
5274 * Tear down any asic specific VM setup (CIK).
5275 */
5276void cik_vm_fini(struct radeon_device *rdev)
5277{
5278}
5279
3ec7d11b
AD		 5280/**
5281 * cik_vm_decode_fault - print human readable fault info
5282 *
5283 * @rdev: radeon_device pointer
5284 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5285 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5286 *
5287 * Print human readable fault information (CIK).
5288 */
5289static void cik_vm_decode_fault(struct radeon_device *rdev,
5290 u32 status, u32 addr, u32 mc_client)
5291{
939c0d3c 5292 u32 mc_id;
3ec7d11b
AD		 5293 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5294 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 5295 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5296 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 5297
939c0d3c
AD		 5298 if (rdev->family == CHIP_HAWAII)
5299 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5300 else
5301 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5302
328a50c7 5303 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 5304 protections, vmid, addr,
5305 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 5306 block, mc_client, mc_id);
3ec7d11b
AD		 5307}
5308
f96ab484
AD		 5309/**
5310 * cik_vm_flush - cik vm flush using the CP
5311 *
5312 * @rdev: radeon_device pointer
5313 *
5314 * Update the page table base and flush the VM TLB
5315 * using the CP (CIK).
5316 */
5317void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5318{
5319 struct radeon_ring *ring = &rdev->ring[ridx];
5320
5321 if (vm == NULL)
5322 return;
5323
5324 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5325 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5326 WRITE_DATA_DST_SEL(0)));
5327 if (vm->id < 8) {
5328 radeon_ring_write(ring,
5329 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5330 } else {
5331 radeon_ring_write(ring,
5332 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5333 }
5334 radeon_ring_write(ring, 0);
5335 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5336
5337 /* update SH_MEM_* regs */
5338 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5339 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5340 WRITE_DATA_DST_SEL(0)));
5341 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5342 radeon_ring_write(ring, 0);
5343 radeon_ring_write(ring, VMID(vm->id));
5344
5345 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5346 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5347 WRITE_DATA_DST_SEL(0)));
5348 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5349 radeon_ring_write(ring, 0);
5350
5351 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5352 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5353 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5354 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 5355
f96ab484
AD		 5356 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5357 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5358 WRITE_DATA_DST_SEL(0)));
5359 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5360 radeon_ring_write(ring, 0);
5361 radeon_ring_write(ring, VMID(0));
6f2043ce 5362
f96ab484
AD		 5363 /* HDP flush */
5364 /* We should be using the WAIT_REG_MEM packet here like in
5365 * cik_fence_ring_emit(), but it causes the CP to hang in this
5366 * context...
5367 */
5368 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5369 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5370 WRITE_DATA_DST_SEL(0)));
5371 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5372 radeon_ring_write(ring, 0);
5373 radeon_ring_write(ring, 0);
5374
5375 /* bits 0-15 are the VM contexts0-15 */
5376 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5377 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5378 WRITE_DATA_DST_SEL(0)));
5379 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5380 radeon_ring_write(ring, 0);
5381 radeon_ring_write(ring, 1 << vm->id);
5382
b07fdd38
AD		 5383 /* compute doesn't have PFP */
5384 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5385 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5386 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5387 radeon_ring_write(ring, 0x0);
5388 }
cc066715 5389}
6f2043ce 5390
f6796cae
AD		 5391/*
5392 * RLC
5393 * The RLC is a multi-purpose microengine that handles a
5394 * variety of functions, the most important of which is
5395 * the interrupt controller.
5396 */
866d83de
AD		 5397static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5398 bool enable)
f6796cae 5399{
866d83de 5400 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 5401
866d83de
AD		 5402 if (enable)
5403 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5404 else
5405 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 5406 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 5407}
f6796cae 5408
866d83de 5409static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 5410{
cc066715 5411 u32 tmp;
6f2043ce 5412
866d83de
AD		 5413 tmp = RREG32(RLC_LB_CNTL);
5414 if (enable)
5415 tmp |= LOAD_BALANCE_ENABLE;
5416 else
5417 tmp &= ~LOAD_BALANCE_ENABLE;
5418 WREG32(RLC_LB_CNTL, tmp);
5419}
cc066715 5420
866d83de
AD		 5421static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5422{
5423 u32 i, j, k;
5424 u32 mask;
cc066715 5425
f6796cae
AD		 5426 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5427 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5428 cik_select_se_sh(rdev, i, j);
5429 for (k = 0; k < rdev->usec_timeout; k++) {
5430 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5431 break;
5432 udelay(1);
5433 }
5434 }
5435 }
5436 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
cc066715 5437
f6796cae
AD		 5438 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5439 for (k = 0; k < rdev->usec_timeout; k++) {
5440 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5441 break;
5442 udelay(1);
5443 }
5444}
cc066715 5445
22c775ce
AD		 5446static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5447{
5448 u32 tmp;
cc066715 5449
22c775ce
AD		 5450 tmp = RREG32(RLC_CNTL);
5451 if (tmp != rlc)
5452 WREG32(RLC_CNTL, rlc);
5453}
cc066715 5454
22c775ce
AD		 5455static u32 cik_halt_rlc(struct radeon_device *rdev)
5456{
5457 u32 data, orig;
cc066715 5458
22c775ce 5459 orig = data = RREG32(RLC_CNTL);
cc066715 5460
22c775ce
AD		 5461 if (data & RLC_ENABLE) {
5462 u32 i;
cc066715 5463
22c775ce
AD		 5464 data &= ~RLC_ENABLE;
5465 WREG32(RLC_CNTL, data);
cc066715 5466
22c775ce
AD		 5467 for (i = 0; i < rdev->usec_timeout; i++) {
5468 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5469 break;
5470 udelay(1);
5471 }
cc066715 5472
22c775ce
AD		 5473 cik_wait_for_rlc_serdes(rdev);
5474 }
cc066715 5475
22c775ce
AD		 5476 return orig;
5477}
cc066715 5478
a412fce0
AD		 5479void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5480{
5481 u32 tmp, i, mask;
5482
5483 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5484 WREG32(RLC_GPR_REG2, tmp);
5485
5486 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5487 for (i = 0; i < rdev->usec_timeout; i++) {
5488 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5489 break;
5490 udelay(1);
5491 }
5492
5493 for (i = 0; i < rdev->usec_timeout; i++) {
5494 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5495 break;
5496 udelay(1);
5497 }
5498}
5499
5500void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5501{
5502 u32 tmp;
5503
5504 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5505 WREG32(RLC_GPR_REG2, tmp);
5506}
5507
866d83de
AD		 5508/**
5509 * cik_rlc_stop - stop the RLC ME
5510 *
5511 * @rdev: radeon_device pointer
5512 *
5513 * Halt the RLC ME (MicroEngine) (CIK).
5514 */
5515static void cik_rlc_stop(struct radeon_device *rdev)
5516{
22c775ce 5517 WREG32(RLC_CNTL, 0);
866d83de
AD		 5518
5519 cik_enable_gui_idle_interrupt(rdev, false);
5520
866d83de
AD		 5521 cik_wait_for_rlc_serdes(rdev);
5522}
5523
f6796cae
AD		 5524/**
5525 * cik_rlc_start - start the RLC ME
5526 *
5527 * @rdev: radeon_device pointer
5528 *
5529 * Unhalt the RLC ME (MicroEngine) (CIK).
5530 */
5531static void cik_rlc_start(struct radeon_device *rdev)
5532{
f6796cae 5533 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 5534
866d83de 5535 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5536
f6796cae 5537 udelay(50);
6f2043ce
AD		 5538}
5539
5540/**
f6796cae 5541 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 5542 *
5543 * @rdev: radeon_device pointer
5544 *
f6796cae
AD		 5545 * Initialize the RLC registers, load the ucode,
5546 * and start the RLC (CIK).
5547 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 5548 */
f6796cae 5549static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 5550{
22c775ce 5551 u32 i, size, tmp;
f6796cae 5552 const __be32 *fw_data;
cc066715 5553
f6796cae
AD		 5554 if (!rdev->rlc_fw)
5555 return -EINVAL;
cc066715 5556
f6796cae
AD		 5557 switch (rdev->family) {
5558 case CHIP_BONAIRE:
d4775655 5559 case CHIP_HAWAII:
f6796cae
AD		 5560 default:
5561 size = BONAIRE_RLC_UCODE_SIZE;
5562 break;
5563 case CHIP_KAVERI:
5564 size = KV_RLC_UCODE_SIZE;
5565 break;
5566 case CHIP_KABINI:
5567 size = KB_RLC_UCODE_SIZE;
5568 break;
5569 }
cc066715 5570
cc066715
AD		 5571 cik_rlc_stop(rdev);
5572
22c775ce
AD		 5573 /* disable CG */
5574 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5575 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 5576
866d83de 5577 si_rlc_reset(rdev);
6f2043ce 5578
22c775ce 5579 cik_init_pg(rdev);
6f2043ce 5580
22c775ce 5581 cik_init_cg(rdev);
cc066715 5582
f6796cae
AD		 5583 WREG32(RLC_LB_CNTR_INIT, 0);
5584 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 5585
f6796cae
AD		 5586 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5587 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5588 WREG32(RLC_LB_PARAMS, 0x00600408);
5589 WREG32(RLC_LB_CNTL, 0x80000004);
cc066715 5590
f6796cae
AD		 5591 WREG32(RLC_MC_CNTL, 0);
5592 WREG32(RLC_UCODE_CNTL, 0);
cc066715 5593
f6796cae
AD		 5594 fw_data = (const __be32 *)rdev->rlc_fw->data;
5595 WREG32(RLC_GPM_UCODE_ADDR, 0);
5596 for (i = 0; i < size; i++)
5597 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5598 WREG32(RLC_GPM_UCODE_ADDR, 0);
cc066715 5599
866d83de
AD		 5600 /* XXX - find out what chips support lbpw */
5601 cik_enable_lbpw(rdev, false);
cc066715 5602
22c775ce
AD		 5603 if (rdev->family == CHIP_BONAIRE)
5604 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 5605
f6796cae 5606 cik_rlc_start(rdev);
cc066715 5607
f6796cae
AD		 5608 return 0;
5609}
cc066715 5610
22c775ce
AD		 5611static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5612{
5613 u32 data, orig, tmp, tmp2;
cc066715 5614
22c775ce 5615 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 5616
473359bc 5617 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 5618 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5619
22c775ce 5620 tmp = cik_halt_rlc(rdev);
cc066715 5621
22c775ce
AD		 5622 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5623 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5624 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5625 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5626 WREG32(RLC_SERDES_WR_CTRL, tmp2);
cc066715 5627
22c775ce 5628 cik_update_rlc(rdev, tmp);
cc066715 5629
22c775ce
AD		 5630 data |= CGCG_EN | CGLS_EN;
5631 } else {
ddc76ff6 5632 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 5633
22c775ce
AD		 5634 RREG32(CB_CGTT_SCLK_CTRL);
5635 RREG32(CB_CGTT_SCLK_CTRL);
5636 RREG32(CB_CGTT_SCLK_CTRL);
5637 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 5638
22c775ce 5639 data &= ~(CGCG_EN | CGLS_EN);
cc066715 5640 }
6f2043ce 5641
22c775ce
AD		 5642 if (orig != data)
5643 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 5644
6f2043ce
AD		 5645}
5646
22c775ce 5647static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 5648{
22c775ce
AD		 5649 u32 data, orig, tmp = 0;
5650
473359bc
AD		 5651 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5652 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5653 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5654 orig = data = RREG32(CP_MEM_SLP_CNTL);
5655 data |= CP_MEM_LS_EN;
5656 if (orig != data)
5657 WREG32(CP_MEM_SLP_CNTL, data);
5658 }
5659 }
cc066715 5660
22c775ce
AD		 5661 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5662 data &= 0xfffffffd;
5663 if (orig != data)
5664 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5665
5666 tmp = cik_halt_rlc(rdev);
5667
5668 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5669 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5670 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5671 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5672 WREG32(RLC_SERDES_WR_CTRL, data);
5673
5674 cik_update_rlc(rdev, tmp);
5675
473359bc
AD		 5676 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5677 orig = data = RREG32(CGTS_SM_CTRL_REG);
5678 data &= ~SM_MODE_MASK;
5679 data |= SM_MODE(0x2);
5680 data |= SM_MODE_ENABLE;
5681 data &= ~CGTS_OVERRIDE;
5682 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5683 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5684 data &= ~CGTS_LS_OVERRIDE;
5685 data &= ~ON_MONITOR_ADD_MASK;
5686 data |= ON_MONITOR_ADD_EN;
5687 data |= ON_MONITOR_ADD(0x96);
5688 if (orig != data)
5689 WREG32(CGTS_SM_CTRL_REG, data);
5690 }
22c775ce
AD		 5691 } else {
5692 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5693 data |= 0x00000002;
5694 if (orig != data)
5695 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5696
5697 data = RREG32(RLC_MEM_SLP_CNTL);
5698 if (data & RLC_MEM_LS_EN) {
5699 data &= ~RLC_MEM_LS_EN;
5700 WREG32(RLC_MEM_SLP_CNTL, data);
5701 }
6f2043ce 5702
22c775ce
AD		 5703 data = RREG32(CP_MEM_SLP_CNTL);
5704 if (data & CP_MEM_LS_EN) {
5705 data &= ~CP_MEM_LS_EN;
5706 WREG32(CP_MEM_SLP_CNTL, data);
5707 }
cc066715 5708
22c775ce
AD		 5709 orig = data = RREG32(CGTS_SM_CTRL_REG);
5710 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5711 if (orig != data)
5712 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 5713
22c775ce 5714 tmp = cik_halt_rlc(rdev);
cc066715 5715
22c775ce
AD		 5716 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5717 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5718 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5719 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5720 WREG32(RLC_SERDES_WR_CTRL, data);
cc066715 5721
22c775ce 5722 cik_update_rlc(rdev, tmp);
cc066715 5723 }
6f2043ce 5724}
1c49165d 5725
22c775ce 5726static const u32 mc_cg_registers[] =
21a93e13 5727{
22c775ce
AD		 5728 MC_HUB_MISC_HUB_CG,
5729 MC_HUB_MISC_SIP_CG,
5730 MC_HUB_MISC_VM_CG,
5731 MC_XPB_CLK_GAT,
5732 ATC_MISC_CG,
5733 MC_CITF_MISC_WR_CG,
5734 MC_CITF_MISC_RD_CG,
5735 MC_CITF_MISC_VM_CG,
5736 VM_L2_CG,
5737};
21a93e13 5738
22c775ce
AD		 5739static void cik_enable_mc_ls(struct radeon_device *rdev,
5740 bool enable)
1c49165d 5741{
22c775ce
AD		 5742 int i;
5743 u32 orig, data;
1c49165d 5744
22c775ce
AD		 5745 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5746 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5747 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 5748 data |= MC_LS_ENABLE;
5749 else
5750 data &= ~MC_LS_ENABLE;
5751 if (data != orig)
5752 WREG32(mc_cg_registers[i], data);
1c49165d 5753 }
22c775ce 5754}
1c49165d 5755
22c775ce
AD		 5756static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5757 bool enable)
5758{
5759 int i;
5760 u32 orig, data;
5761
5762 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5763 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5764 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 5765 data |= MC_CG_ENABLE;
5766 else
5767 data &= ~MC_CG_ENABLE;
5768 if (data != orig)
5769 WREG32(mc_cg_registers[i], data);
1c49165d 5770 }
1c49165d
AD		 5771}
5772
22c775ce
AD		 5773static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5774 bool enable)
1c49165d 5775{
22c775ce 5776 u32 orig, data;
1c49165d 5777
473359bc 5778 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 5779 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5780 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 5781 } else {
22c775ce
AD		 5782 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5783 data |= 0xff000000;
5784 if (data != orig)
5785 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 5786
22c775ce
AD		 5787 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5788 data |= 0xff000000;
5789 if (data != orig)
5790 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5791 }
1c49165d
AD		 5792}
5793
22c775ce
AD		 5794static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5795 bool enable)
1c49165d 5796{
22c775ce
AD		 5797 u32 orig, data;
5798
473359bc 5799 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 5800 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5801 data |= 0x100;
5802 if (orig != data)
5803 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5804
5805 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5806 data |= 0x100;
5807 if (orig != data)
5808 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5809 } else {
5810 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5811 data &= ~0x100;
5812 if (orig != data)
5813 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 5814
22c775ce
AD		 5815 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5816 data &= ~0x100;
5817 if (orig != data)
5818 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5819 }
1c49165d
AD		 5820}
5821
22c775ce
AD		 5822static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5823 bool enable)
1c49165d 5824{
22c775ce 5825 u32 orig, data;
1c49165d 5826
473359bc 5827 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 5828 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5829 data = 0xfff;
5830 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 5831
22c775ce
AD		 5832 orig = data = RREG32(UVD_CGC_CTRL);
5833 data |= DCM;
5834 if (orig != data)
5835 WREG32(UVD_CGC_CTRL, data);
5836 } else {
5837 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5838 data &= ~0xfff;
5839 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 5840
22c775ce
AD		 5841 orig = data = RREG32(UVD_CGC_CTRL);
5842 data &= ~DCM;
5843 if (orig != data)
5844 WREG32(UVD_CGC_CTRL, data);
1c49165d 5845 }
22c775ce 5846}
1c49165d 5847
473359bc
AD		 5848static void cik_enable_bif_mgls(struct radeon_device *rdev,
5849 bool enable)
5850{
5851 u32 orig, data;
1c49165d 5852
473359bc 5853 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 5854
473359bc
AD		 5855 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5856 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5857 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5858 else
5859 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5860 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 5861
473359bc
AD		 5862 if (orig != data)
5863 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5864}
1c49165d 5865
22c775ce
AD		 5866static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5867 bool enable)
5868{
5869 u32 orig, data;
1c49165d 5870
22c775ce 5871 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 5872
473359bc 5873 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 5874 data &= ~CLOCK_GATING_DIS;
5875 else
5876 data |= CLOCK_GATING_DIS;
5877
5878 if (orig != data)
5879 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 5880}
5881
22c775ce
AD		 5882static void cik_enable_hdp_ls(struct radeon_device *rdev,
5883 bool enable)
1c49165d 5884{
22c775ce
AD		 5885 u32 orig, data;
5886
5887 orig = data = RREG32(HDP_MEM_POWER_LS);
5888
473359bc 5889 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 5890 data |= HDP_LS_ENABLE;
5891 else
5892 data &= ~HDP_LS_ENABLE;
5893
5894 if (orig != data)
5895 WREG32(HDP_MEM_POWER_LS, data);
5896}
5897
5898void cik_update_cg(struct radeon_device *rdev,
5899 u32 block, bool enable)
5900{
4214faf6 5901
22c775ce 5902 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 5903 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 5904 /* order matters! */
5905 if (enable) {
5906 cik_enable_mgcg(rdev, true);
5907 cik_enable_cgcg(rdev, true);
5908 } else {
5909 cik_enable_cgcg(rdev, false);
5910 cik_enable_mgcg(rdev, false);
5911 }
4214faf6 5912 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 5913 }
5914
5915 if (block & RADEON_CG_BLOCK_MC) {
5916 if (!(rdev->flags & RADEON_IS_IGP)) {
5917 cik_enable_mc_mgcg(rdev, enable);
5918 cik_enable_mc_ls(rdev, enable);
5919 }
5920 }
5921
5922 if (block & RADEON_CG_BLOCK_SDMA) {
5923 cik_enable_sdma_mgcg(rdev, enable);
5924 cik_enable_sdma_mgls(rdev, enable);
5925 }
5926
473359bc
AD		 5927 if (block & RADEON_CG_BLOCK_BIF) {
5928 cik_enable_bif_mgls(rdev, enable);
5929 }
5930
22c775ce
AD		 5931 if (block & RADEON_CG_BLOCK_UVD) {
5932 if (rdev->has_uvd)
5933 cik_enable_uvd_mgcg(rdev, enable);
5934 }
5935
5936 if (block & RADEON_CG_BLOCK_HDP) {
5937 cik_enable_hdp_mgcg(rdev, enable);
5938 cik_enable_hdp_ls(rdev, enable);
5939 }
1c49165d
AD		 5940}
5941
22c775ce 5942static void cik_init_cg(struct radeon_device *rdev)
1c49165d 5943{
22c775ce 5944
ddc76ff6 5945 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 5946
5947 if (rdev->has_uvd)
5948 si_init_uvd_internal_cg(rdev);
5949
5950 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5951 RADEON_CG_BLOCK_SDMA |
473359bc 5952 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 5953 RADEON_CG_BLOCK_UVD |
5954 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 5955}
5956
473359bc 5957static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 5958{
473359bc
AD		 5959 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5960 RADEON_CG_BLOCK_SDMA |
5961 RADEON_CG_BLOCK_BIF |
5962 RADEON_CG_BLOCK_UVD |
5963 RADEON_CG_BLOCK_HDP), false);
5964
5965 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 5966}
5967
22c775ce
AD		 5968static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5969 bool enable)
1c49165d 5970{
22c775ce 5971 u32 data, orig;
1c49165d 5972
22c775ce 5973 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5974 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 5975 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5976 else
5977 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5978 if (orig != data)
5979 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 5980}
5981
22c775ce
AD		 5982static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5983 bool enable)
1c49165d 5984{
22c775ce
AD		 5985 u32 data, orig;
5986
5987 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5988 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 5989 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5990 else
5991 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5992 if (orig != data)
5993 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 5994}
5995
22c775ce 5996static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 5997{
22c775ce 5998 u32 data, orig;
3ec7d11b 5999
22c775ce 6000 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6001 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6002 data &= ~DISABLE_CP_PG;
6003 else
6004 data |= DISABLE_CP_PG;
6005 if (orig != data)
6006 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6007}
6008
22c775ce 6009static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6010{
22c775ce 6011 u32 data, orig;
f96ab484 6012
22c775ce 6013 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6014 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6015 data &= ~DISABLE_GDS_PG;
6016 else
6017 data |= DISABLE_GDS_PG;
6018 if (orig != data)
6019 WREG32(RLC_PG_CNTL, data);
6020}
6021
6022#define CP_ME_TABLE_SIZE 96
6023#define CP_ME_TABLE_OFFSET 2048
6024#define CP_MEC_TABLE_OFFSET 4096
6025
6026void cik_init_cp_pg_table(struct radeon_device *rdev)
6027{
6028 const __be32 *fw_data;
6029 volatile u32 *dst_ptr;
6030 int me, i, max_me = 4;
6031 u32 bo_offset = 0;
6032 u32 table_offset;
6033
6034 if (rdev->family == CHIP_KAVERI)
6035 max_me = 5;
6036
6037 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6038 return;
6039
22c775ce
AD		 6040 /* write the cp table buffer */
6041 dst_ptr = rdev->rlc.cp_table_ptr;
6042 for (me = 0; me < max_me; me++) {
6043 if (me == 0) {
6044 fw_data = (const __be32 *)rdev->ce_fw->data;
6045 table_offset = CP_ME_TABLE_OFFSET;
6046 } else if (me == 1) {
6047 fw_data = (const __be32 *)rdev->pfp_fw->data;
6048 table_offset = CP_ME_TABLE_OFFSET;
6049 } else if (me == 2) {
6050 fw_data = (const __be32 *)rdev->me_fw->data;
6051 table_offset = CP_ME_TABLE_OFFSET;
6052 } else {
6053 fw_data = (const __be32 *)rdev->mec_fw->data;
6054 table_offset = CP_MEC_TABLE_OFFSET;
6055 }
6056
6057 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6ba81e53 6058 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
22c775ce
AD		 6059 }
6060 bo_offset += CP_ME_TABLE_SIZE;
f96ab484 6061 }
22c775ce 6062}
f96ab484 6063
22c775ce
AD		 6064static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6065 bool enable)
6066{
6067 u32 data, orig;
6068
2b19d17f 6069 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6070 orig = data = RREG32(RLC_PG_CNTL);
6071 data |= GFX_PG_ENABLE;
6072 if (orig != data)
6073 WREG32(RLC_PG_CNTL, data);
6074
6075 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6076 data |= AUTO_PG_EN;
6077 if (orig != data)
6078 WREG32(RLC_AUTO_PG_CTRL, data);
6079 } else {
6080 orig = data = RREG32(RLC_PG_CNTL);
6081 data &= ~GFX_PG_ENABLE;
6082 if (orig != data)
6083 WREG32(RLC_PG_CNTL, data);
f96ab484 6084
22c775ce
AD		 6085 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6086 data &= ~AUTO_PG_EN;
6087 if (orig != data)
6088 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6089
22c775ce
AD		 6090 data = RREG32(DB_RENDER_CONTROL);
6091 }
6092}
f96ab484 6093
22c775ce
AD		 6094static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6095{
6096 u32 mask = 0, tmp, tmp1;
6097 int i;
f96ab484 6098
22c775ce
AD		 6099 cik_select_se_sh(rdev, se, sh);
6100 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6101 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6102 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
f96ab484 6103
22c775ce 6104 tmp &= 0xffff0000;
f96ab484 6105
22c775ce
AD		 6106 tmp |= tmp1;
6107 tmp >>= 16;
6108
6109 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6110 mask <<= 1;
6111 mask |= 1;
b07fdd38 6112 }
22c775ce
AD		 6113
6114 return (~tmp) & mask;
f96ab484
AD		 6115}
6116
22c775ce 6117static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6118{
22c775ce
AD		 6119 u32 i, j, k, active_cu_number = 0;
6120 u32 mask, counter, cu_bitmap;
6121 u32 tmp = 0;
d0e092d9 6122
22c775ce
AD		 6123 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6124 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6125 mask = 1;
6126 cu_bitmap = 0;
6127 counter = 0;
6128 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6129 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6130 if (counter < 2)
6131 cu_bitmap |= mask;
6132 counter ++;
d0e092d9 6133 }
22c775ce 6134 mask <<= 1;
d0e092d9 6135 }
d0e092d9 6136
22c775ce
AD		 6137 active_cu_number += counter;
6138 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6139 }
d0e092d9 6140 }
22c775ce
AD		 6141
6142 WREG32(RLC_PG_AO_CU_MASK, tmp);
6143
6144 tmp = RREG32(RLC_MAX_PG_CU);
6145 tmp &= ~MAX_PU_CU_MASK;
6146 tmp |= MAX_PU_CU(active_cu_number);
6147 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6148}
6149
22c775ce
AD		 6150static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6151 bool enable)
605de6b9 6152{
22c775ce 6153 u32 data, orig;
605de6b9 6154
22c775ce 6155 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6156 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6157 data |= STATIC_PER_CU_PG_ENABLE;
6158 else
6159 data &= ~STATIC_PER_CU_PG_ENABLE;
6160 if (orig != data)
6161 WREG32(RLC_PG_CNTL, data);
6162}
6163
6164static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6165 bool enable)
6166{
6167 u32 data, orig;
605de6b9 6168
22c775ce 6169 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6170 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6171 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6172 else
22c775ce
AD		 6173 data &= ~DYN_PER_CU_PG_ENABLE;
6174 if (orig != data)
6175 WREG32(RLC_PG_CNTL, data);
6176}
605de6b9 6177
22c775ce
AD		 6178#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6179#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6180
6181static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6182{
6183 u32 data, orig;
6184 u32 i;
6185
6186 if (rdev->rlc.cs_data) {
6187 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6188 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 6189 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 6190 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 6191 } else {
22c775ce
AD		 6192 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6193 for (i = 0; i < 3; i++)
6194 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6195 }
6196 if (rdev->rlc.reg_list) {
6197 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6198 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6199 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 6200 }
605de6b9 6201
22c775ce
AD		 6202 orig = data = RREG32(RLC_PG_CNTL);
6203 data |= GFX_PG_SRC;
6204 if (orig != data)
6205 WREG32(RLC_PG_CNTL, data);
605de6b9 6206
22c775ce
AD		 6207 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6208 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 6209
22c775ce
AD		 6210 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6211 data &= ~IDLE_POLL_COUNT_MASK;
6212 data |= IDLE_POLL_COUNT(0x60);
6213 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 6214
22c775ce
AD		 6215 data = 0x10101010;
6216 WREG32(RLC_PG_DELAY, data);
605de6b9 6217
22c775ce
AD		 6218 data = RREG32(RLC_PG_DELAY_2);
6219 data &= ~0xff;
6220 data |= 0x3;
6221 WREG32(RLC_PG_DELAY_2, data);
605de6b9 6222
22c775ce
AD		 6223 data = RREG32(RLC_AUTO_PG_CTRL);
6224 data &= ~GRBM_REG_SGIT_MASK;
6225 data |= GRBM_REG_SGIT(0x700);
6226 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 6227
605de6b9
AD		 6228}
6229
22c775ce 6230static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 6231{
473359bc
AD		 6232 cik_enable_gfx_cgpg(rdev, enable);
6233 cik_enable_gfx_static_mgpg(rdev, enable);
6234 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 6235}
f6796cae 6236
a0f38609
AD		 6237u32 cik_get_csb_size(struct radeon_device *rdev)
6238{
6239 u32 count = 0;
6240 const struct cs_section_def *sect = NULL;
6241 const struct cs_extent_def *ext = NULL;
f6796cae 6242
a0f38609
AD		 6243 if (rdev->rlc.cs_data == NULL)
6244 return 0;
f6796cae 6245
a0f38609
AD		 6246 /* begin clear state */
6247 count += 2;
6248 /* context control state */
6249 count += 3;
6250
6251 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6252 for (ext = sect->section; ext->extent != NULL; ++ext) {
6253 if (sect->id == SECT_CONTEXT)
6254 count += 2 + ext->reg_count;
6255 else
6256 return 0;
f6796cae
AD		 6257 }
6258 }
a0f38609
AD		 6259 /* pa_sc_raster_config/pa_sc_raster_config1 */
6260 count += 4;
6261 /* end clear state */
6262 count += 2;
6263 /* clear state */
6264 count += 2;
f6796cae 6265
a0f38609 6266 return count;
f6796cae
AD		 6267}
6268
a0f38609 6269void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 6270{
a0f38609
AD		 6271 u32 count = 0, i;
6272 const struct cs_section_def *sect = NULL;
6273 const struct cs_extent_def *ext = NULL;
f6796cae 6274
a0f38609
AD		 6275 if (rdev->rlc.cs_data == NULL)
6276 return;
6277 if (buffer == NULL)
6278 return;
f6796cae 6279
6ba81e53
AD		 6280 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6281 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 6282
6ba81e53
AD		 6283 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6284 buffer[count++] = cpu_to_le32(0x80000000);
6285 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 6286
6287 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6288 for (ext = sect->section; ext->extent != NULL; ++ext) {
6289 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 6290 buffer[count++] =
6291 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6292 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 6293 for (i = 0; i < ext->reg_count; i++)
6ba81e53 6294 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 6295 } else {
6296 return;
6297 }
6298 }
6299 }
f6796cae 6300
6ba81e53
AD		 6301 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6302 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 6303 switch (rdev->family) {
6304 case CHIP_BONAIRE:
6ba81e53
AD		 6305 buffer[count++] = cpu_to_le32(0x16000012);
6306 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6307 break;
6308 case CHIP_KAVERI:
6ba81e53
AD		 6309 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6310 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6311 break;
6312 case CHIP_KABINI:
6ba81e53
AD		 6313 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6314 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 6315 break;
bbfe90bd
AD		 6316 case CHIP_HAWAII:
6317 buffer[count++] = 0x3a00161a;
6318 buffer[count++] = 0x0000002e;
6319 break;
a0f38609 6320 default:
6ba81e53
AD		 6321 buffer[count++] = cpu_to_le32(0x00000000);
6322 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6323 break;
6324 }
6325
6ba81e53
AD		 6326 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6327 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 6328
6ba81e53
AD		 6329 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6330 buffer[count++] = cpu_to_le32(0);
a0f38609 6331}
f6796cae 6332
473359bc 6333static void cik_init_pg(struct radeon_device *rdev)
22c775ce 6334{
473359bc 6335 if (rdev->pg_flags) {
22c775ce
AD		 6336 cik_enable_sck_slowdown_on_pu(rdev, true);
6337 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 6338 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6339 cik_init_gfx_cgpg(rdev);
6340 cik_enable_cp_pg(rdev, true);
6341 cik_enable_gds_pg(rdev, true);
6342 }
22c775ce
AD		 6343 cik_init_ao_cu_mask(rdev);
6344 cik_update_gfx_pg(rdev, true);
6345 }
6346}
f6796cae 6347
473359bc
AD		 6348static void cik_fini_pg(struct radeon_device *rdev)
6349{
6350 if (rdev->pg_flags) {
6351 cik_update_gfx_pg(rdev, false);
2b19d17f 6352 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6353 cik_enable_cp_pg(rdev, false);
6354 cik_enable_gds_pg(rdev, false);
6355 }
6356 }
f6796cae 6357}
a59781bb
AD		 6358
6359/*
6360 * Interrupts
6361 * Starting with r6xx, interrupts are handled via a ring buffer.
6362 * Ring buffers are areas of GPU accessible memory that the GPU
6363 * writes interrupt vectors into and the host reads vectors out of.
6364 * There is a rptr (read pointer) that determines where the
6365 * host is currently reading, and a wptr (write pointer)
6366 * which determines where the GPU has written. When the
6367 * pointers are equal, the ring is idle. When the GPU
6368 * writes vectors to the ring buffer, it increments the
6369 * wptr. When there is an interrupt, the host then starts
6370 * fetching commands and processing them until the pointers are
6371 * equal again at which point it updates the rptr.
6372 */
6373
6374/**
6375 * cik_enable_interrupts - Enable the interrupt ring buffer
6376 *
6377 * @rdev: radeon_device pointer
6378 *
6379 * Enable the interrupt ring buffer (CIK).
6380 */
6381static void cik_enable_interrupts(struct radeon_device *rdev)
6382{
6383 u32 ih_cntl = RREG32(IH_CNTL);
6384 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6385
6386 ih_cntl |= ENABLE_INTR;
6387 ih_rb_cntl |= IH_RB_ENABLE;
6388 WREG32(IH_CNTL, ih_cntl);
6389 WREG32(IH_RB_CNTL, ih_rb_cntl);
6390 rdev->ih.enabled = true;
6391}
6392
6393/**
6394 * cik_disable_interrupts - Disable the interrupt ring buffer
6395 *
6396 * @rdev: radeon_device pointer
6397 *
6398 * Disable the interrupt ring buffer (CIK).
6399 */
6400static void cik_disable_interrupts(struct radeon_device *rdev)
6401{
6402 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6403 u32 ih_cntl = RREG32(IH_CNTL);
6404
6405 ih_rb_cntl &= ~IH_RB_ENABLE;
6406 ih_cntl &= ~ENABLE_INTR;
6407 WREG32(IH_RB_CNTL, ih_rb_cntl);
6408 WREG32(IH_CNTL, ih_cntl);
6409 /* set rptr, wptr to 0 */
6410 WREG32(IH_RB_RPTR, 0);
6411 WREG32(IH_RB_WPTR, 0);
6412 rdev->ih.enabled = false;
6413 rdev->ih.rptr = 0;
6414}
6415
6416/**
6417 * cik_disable_interrupt_state - Disable all interrupt sources
6418 *
6419 * @rdev: radeon_device pointer
6420 *
6421 * Clear all interrupt enable bits used by the driver (CIK).
6422 */
6423static void cik_disable_interrupt_state(struct radeon_device *rdev)
6424{
6425 u32 tmp;
6426
6427 /* gfx ring */
4214faf6
AD		 6428 tmp = RREG32(CP_INT_CNTL_RING0) &
6429 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6430 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 6431 /* sdma */
6432 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6433 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6434 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6435 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 6436 /* compute queues */
6437 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6438 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6439 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6440 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6441 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6442 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6443 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6444 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6445 /* grbm */
6446 WREG32(GRBM_INT_CNTL, 0);
6447 /* vline/vblank, etc. */
6448 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6449 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6450 if (rdev->num_crtc >= 4) {
6451 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6452 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6453 }
6454 if (rdev->num_crtc >= 6) {
6455 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6456 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6457 }
6458
6459 /* dac hotplug */
6460 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6461
6462 /* digital hotplug */
6463 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6464 WREG32(DC_HPD1_INT_CONTROL, tmp);
6465 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6466 WREG32(DC_HPD2_INT_CONTROL, tmp);
6467 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6468 WREG32(DC_HPD3_INT_CONTROL, tmp);
6469 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6470 WREG32(DC_HPD4_INT_CONTROL, tmp);
6471 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6472 WREG32(DC_HPD5_INT_CONTROL, tmp);
6473 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6474 WREG32(DC_HPD6_INT_CONTROL, tmp);
6475
6476}
6477
6478/**
6479 * cik_irq_init - init and enable the interrupt ring
6480 *
6481 * @rdev: radeon_device pointer
6482 *
6483 * Allocate a ring buffer for the interrupt controller,
6484 * enable the RLC, disable interrupts, enable the IH
6485 * ring buffer and enable it (CIK).
6486 * Called at device load and reume.
6487 * Returns 0 for success, errors for failure.
6488 */
6489static int cik_irq_init(struct radeon_device *rdev)
6490{
6491 int ret = 0;
6492 int rb_bufsz;
6493 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6494
6495 /* allocate ring */
6496 ret = r600_ih_ring_alloc(rdev);
6497 if (ret)
6498 return ret;
6499
6500 /* disable irqs */
6501 cik_disable_interrupts(rdev);
6502
6503 /* init rlc */
6504 ret = cik_rlc_resume(rdev);
6505 if (ret) {
6506 r600_ih_ring_fini(rdev);
6507 return ret;
6508 }
6509
6510 /* setup interrupt control */
6511 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6512 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6513 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6514 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6515 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6516 */
6517 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6518 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6519 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6520 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6521
6522 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 6523 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 6524
6525 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6526 IH_WPTR_OVERFLOW_CLEAR |
6527 (rb_bufsz << 1));
6528
6529 if (rdev->wb.enabled)
6530 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6531
6532 /* set the writeback address whether it's enabled or not */
6533 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6534 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6535
6536 WREG32(IH_RB_CNTL, ih_rb_cntl);
6537
6538 /* set rptr, wptr to 0 */
6539 WREG32(IH_RB_RPTR, 0);
6540 WREG32(IH_RB_WPTR, 0);
6541
6542 /* Default settings for IH_CNTL (disabled at first) */
6543 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6544 /* RPTR_REARM only works if msi's are enabled */
6545 if (rdev->msi_enabled)
6546 ih_cntl |= RPTR_REARM;
6547 WREG32(IH_CNTL, ih_cntl);
6548
6549 /* force the active interrupt state to all disabled */
6550 cik_disable_interrupt_state(rdev);
6551
6552 pci_set_master(rdev->pdev);
6553
6554 /* enable irqs */
6555 cik_enable_interrupts(rdev);
6556
6557 return ret;
6558}
6559
6560/**
6561 * cik_irq_set - enable/disable interrupt sources
6562 *
6563 * @rdev: radeon_device pointer
6564 *
6565 * Enable interrupt sources on the GPU (vblanks, hpd,
6566 * etc.) (CIK).
6567 * Returns 0 for success, errors for failure.
6568 */
6569int cik_irq_set(struct radeon_device *rdev)
6570{
4214faf6 6571 u32 cp_int_cntl;
2b0781a6
AD		 6572 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6573 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
a59781bb
AD		 6574 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6575 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6576 u32 grbm_int_cntl = 0;
21a93e13 6577 u32 dma_cntl, dma_cntl1;
41a524ab 6578 u32 thermal_int;
a59781bb
AD		 6579
6580 if (!rdev->irq.installed) {
6581 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6582 return -EINVAL;
6583 }
6584 /* don't enable anything if the ih is disabled */
6585 if (!rdev->ih.enabled) {
6586 cik_disable_interrupts(rdev);
6587 /* force the active interrupt state to all disabled */
6588 cik_disable_interrupt_state(rdev);
6589 return 0;
6590 }
6591
4214faf6
AD		 6592 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6593 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6594 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6595
a59781bb
AD		 6596 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6597 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6598 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6599 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6600 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6601 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6602
21a93e13
AD		 6603 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6604 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6605
2b0781a6
AD		 6606 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6607 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6608 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6609 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6610 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6611 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6612 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6613 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6614
cc8dbbb4
AD		 6615 if (rdev->flags & RADEON_IS_IGP)
6616 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6617 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6618 else
6619 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6620 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
41a524ab 6621
a59781bb
AD		 6622 /* enable CP interrupts on all rings */
6623 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6624 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6625 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6626 }
2b0781a6
AD		 6627 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6628 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6629 DRM_DEBUG("si_irq_set: sw int cp1\n");
6630 if (ring->me == 1) {
6631 switch (ring->pipe) {
6632 case 0:
6633 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6634 break;
6635 case 1:
6636 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6637 break;
6638 case 2:
6639 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6640 break;
6641 case 3:
6642 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6643 break;
6644 default:
6645 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6646 break;
6647 }
6648 } else if (ring->me == 2) {
6649 switch (ring->pipe) {
6650 case 0:
6651 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6652 break;
6653 case 1:
6654 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6655 break;
6656 case 2:
6657 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6658 break;
6659 case 3:
6660 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6661 break;
6662 default:
6663 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6664 break;
6665 }
6666 } else {
6667 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6668 }
6669 }
6670 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6671 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6672 DRM_DEBUG("si_irq_set: sw int cp2\n");
6673 if (ring->me == 1) {
6674 switch (ring->pipe) {
6675 case 0:
6676 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6677 break;
6678 case 1:
6679 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6680 break;
6681 case 2:
6682 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6683 break;
6684 case 3:
6685 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6686 break;
6687 default:
6688 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6689 break;
6690 }
6691 } else if (ring->me == 2) {
6692 switch (ring->pipe) {
6693 case 0:
6694 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6695 break;
6696 case 1:
6697 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6698 break;
6699 case 2:
6700 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6701 break;
6702 case 3:
6703 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6704 break;
6705 default:
6706 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6707 break;
6708 }
6709 } else {
6710 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6711 }
6712 }
a59781bb 6713
21a93e13
AD		 6714 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6715 DRM_DEBUG("cik_irq_set: sw int dma\n");
6716 dma_cntl |= TRAP_ENABLE;
6717 }
6718
6719 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6720 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6721 dma_cntl1 |= TRAP_ENABLE;
6722 }
6723
a59781bb
AD		 6724 if (rdev->irq.crtc_vblank_int[0] ||
6725 atomic_read(&rdev->irq.pflip[0])) {
6726 DRM_DEBUG("cik_irq_set: vblank 0\n");
6727 crtc1 |= VBLANK_INTERRUPT_MASK;
6728 }
6729 if (rdev->irq.crtc_vblank_int[1] ||
6730 atomic_read(&rdev->irq.pflip[1])) {
6731 DRM_DEBUG("cik_irq_set: vblank 1\n");
6732 crtc2 |= VBLANK_INTERRUPT_MASK;
6733 }
6734 if (rdev->irq.crtc_vblank_int[2] ||
6735 atomic_read(&rdev->irq.pflip[2])) {
6736 DRM_DEBUG("cik_irq_set: vblank 2\n");
6737 crtc3 |= VBLANK_INTERRUPT_MASK;
6738 }
6739 if (rdev->irq.crtc_vblank_int[3] ||
6740 atomic_read(&rdev->irq.pflip[3])) {
6741 DRM_DEBUG("cik_irq_set: vblank 3\n");
6742 crtc4 |= VBLANK_INTERRUPT_MASK;
6743 }
6744 if (rdev->irq.crtc_vblank_int[4] ||
6745 atomic_read(&rdev->irq.pflip[4])) {
6746 DRM_DEBUG("cik_irq_set: vblank 4\n");
6747 crtc5 |= VBLANK_INTERRUPT_MASK;
6748 }
6749 if (rdev->irq.crtc_vblank_int[5] ||
6750 atomic_read(&rdev->irq.pflip[5])) {
6751 DRM_DEBUG("cik_irq_set: vblank 5\n");
6752 crtc6 |= VBLANK_INTERRUPT_MASK;
6753 }
6754 if (rdev->irq.hpd[0]) {
6755 DRM_DEBUG("cik_irq_set: hpd 1\n");
6756 hpd1 |= DC_HPDx_INT_EN;
6757 }
6758 if (rdev->irq.hpd[1]) {
6759 DRM_DEBUG("cik_irq_set: hpd 2\n");
6760 hpd2 |= DC_HPDx_INT_EN;
6761 }
6762 if (rdev->irq.hpd[2]) {
6763 DRM_DEBUG("cik_irq_set: hpd 3\n");
6764 hpd3 |= DC_HPDx_INT_EN;
6765 }
6766 if (rdev->irq.hpd[3]) {
6767 DRM_DEBUG("cik_irq_set: hpd 4\n");
6768 hpd4 |= DC_HPDx_INT_EN;
6769 }
6770 if (rdev->irq.hpd[4]) {
6771 DRM_DEBUG("cik_irq_set: hpd 5\n");
6772 hpd5 |= DC_HPDx_INT_EN;
6773 }
6774 if (rdev->irq.hpd[5]) {
6775 DRM_DEBUG("cik_irq_set: hpd 6\n");
6776 hpd6 |= DC_HPDx_INT_EN;
6777 }
6778
41a524ab
AD		 6779 if (rdev->irq.dpm_thermal) {
6780 DRM_DEBUG("dpm thermal\n");
cc8dbbb4
AD		 6781 if (rdev->flags & RADEON_IS_IGP)
6782 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6783 else
6784 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
41a524ab
AD		 6785 }
6786
a59781bb
AD		 6787 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6788
21a93e13
AD		 6789 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6790 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6791
2b0781a6
AD		 6792 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6793 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6794 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6795 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6796 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6797 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6798 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6799 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6800
a59781bb
AD		 6801 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6802
6803 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6804 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6805 if (rdev->num_crtc >= 4) {
6806 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6807 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6808 }
6809 if (rdev->num_crtc >= 6) {
6810 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6811 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6812 }
6813
6814 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6815 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6816 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6817 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6818 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6819 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6820
cc8dbbb4
AD		 6821 if (rdev->flags & RADEON_IS_IGP)
6822 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6823 else
6824 WREG32_SMC(CG_THERMAL_INT, thermal_int);
41a524ab 6825
a59781bb
AD		 6826 return 0;
6827}
6828
6829/**
6830 * cik_irq_ack - ack interrupt sources
6831 *
6832 * @rdev: radeon_device pointer
6833 *
6834 * Ack interrupt sources on the GPU (vblanks, hpd,
6835 * etc.) (CIK). Certain interrupts sources are sw
6836 * generated and do not require an explicit ack.
6837 */
6838static inline void cik_irq_ack(struct radeon_device *rdev)
6839{
6840 u32 tmp;
6841
6842 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6843 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6844 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6845 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6846 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6847 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6848 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6849
6850 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6851 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6852 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6853 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6854 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6855 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6856 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6857 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6858
6859 if (rdev->num_crtc >= 4) {
6860 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6861 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6862 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6863 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6864 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6865 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6866 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6867 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6868 }
6869
6870 if (rdev->num_crtc >= 6) {
6871 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6872 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6873 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6874 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6875 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6876 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6877 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6878 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6879 }
6880
6881 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6882 tmp = RREG32(DC_HPD1_INT_CONTROL);
6883 tmp |= DC_HPDx_INT_ACK;
6884 WREG32(DC_HPD1_INT_CONTROL, tmp);
6885 }
6886 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6887 tmp = RREG32(DC_HPD2_INT_CONTROL);
6888 tmp |= DC_HPDx_INT_ACK;
6889 WREG32(DC_HPD2_INT_CONTROL, tmp);
6890 }
6891 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6892 tmp = RREG32(DC_HPD3_INT_CONTROL);
6893 tmp |= DC_HPDx_INT_ACK;
6894 WREG32(DC_HPD3_INT_CONTROL, tmp);
6895 }
6896 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6897 tmp = RREG32(DC_HPD4_INT_CONTROL);
6898 tmp |= DC_HPDx_INT_ACK;
6899 WREG32(DC_HPD4_INT_CONTROL, tmp);
6900 }
6901 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6902 tmp = RREG32(DC_HPD5_INT_CONTROL);
6903 tmp |= DC_HPDx_INT_ACK;
6904 WREG32(DC_HPD5_INT_CONTROL, tmp);
6905 }
6906 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6907 tmp = RREG32(DC_HPD5_INT_CONTROL);
6908 tmp |= DC_HPDx_INT_ACK;
6909 WREG32(DC_HPD6_INT_CONTROL, tmp);
6910 }
6911}
6912
6913/**
6914 * cik_irq_disable - disable interrupts
6915 *
6916 * @rdev: radeon_device pointer
6917 *
6918 * Disable interrupts on the hw (CIK).
6919 */
6920static void cik_irq_disable(struct radeon_device *rdev)
6921{
6922 cik_disable_interrupts(rdev);
6923 /* Wait and acknowledge irq */
6924 mdelay(1);
6925 cik_irq_ack(rdev);
6926 cik_disable_interrupt_state(rdev);
6927}
6928
6929/**
6930 * cik_irq_disable - disable interrupts for suspend
6931 *
6932 * @rdev: radeon_device pointer
6933 *
6934 * Disable interrupts and stop the RLC (CIK).
6935 * Used for suspend.
6936 */
6937static void cik_irq_suspend(struct radeon_device *rdev)
6938{
6939 cik_irq_disable(rdev);
6940 cik_rlc_stop(rdev);
6941}
6942
6943/**
6944 * cik_irq_fini - tear down interrupt support
6945 *
6946 * @rdev: radeon_device pointer
6947 *
6948 * Disable interrupts on the hw and free the IH ring
6949 * buffer (CIK).
6950 * Used for driver unload.
6951 */
6952static void cik_irq_fini(struct radeon_device *rdev)
6953{
6954 cik_irq_suspend(rdev);
6955 r600_ih_ring_fini(rdev);
6956}
6957
6958/**
6959 * cik_get_ih_wptr - get the IH ring buffer wptr
6960 *
6961 * @rdev: radeon_device pointer
6962 *
6963 * Get the IH ring buffer wptr from either the register
6964 * or the writeback memory buffer (CIK). Also check for
6965 * ring buffer overflow and deal with it.
6966 * Used by cik_irq_process().
6967 * Returns the value of the wptr.
6968 */
6969static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6970{
6971 u32 wptr, tmp;
6972
6973 if (rdev->wb.enabled)
6974 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6975 else
6976 wptr = RREG32(IH_RB_WPTR);
6977
6978 if (wptr & RB_OVERFLOW) {
6979 /* When a ring buffer overflow happen start parsing interrupt
6980 * from the last not overwritten vector (wptr + 16). Hopefully
6981 * this should allow us to catchup.
6982 */
6983 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6984 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6985 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6986 tmp = RREG32(IH_RB_CNTL);
6987 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6988 WREG32(IH_RB_CNTL, tmp);
6989 }
6990 return (wptr & rdev->ih.ptr_mask);
6991}
6992
6993/* CIK IV Ring
6994 * Each IV ring entry is 128 bits:
6995 * [7:0] - interrupt source id
6996 * [31:8] - reserved
6997 * [59:32] - interrupt source data
6998 * [63:60] - reserved
21a93e13
AD		 6999 * [71:64] - RINGID
7000 * CP:
7001 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7002 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7003 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7004 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7005 * PIPE_ID - ME0 0=3D
7006 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7007 * SDMA:
7008 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7009 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7010 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7011 * [79:72] - VMID
7012 * [95:80] - PASID
7013 * [127:96] - reserved
7014 */
7015/**
7016 * cik_irq_process - interrupt handler
7017 *
7018 * @rdev: radeon_device pointer
7019 *
7020 * Interrupt hander (CIK). Walk the IH ring,
7021 * ack interrupts and schedule work to handle
7022 * interrupt events.
7023 * Returns irq process return code.
7024 */
7025int cik_irq_process(struct radeon_device *rdev)
7026{
2b0781a6
AD		 7027 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7028 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7029 u32 wptr;
7030 u32 rptr;
7031 u32 src_id, src_data, ring_id;
7032 u8 me_id, pipe_id, queue_id;
7033 u32 ring_index;
7034 bool queue_hotplug = false;
7035 bool queue_reset = false;
3ec7d11b 7036 u32 addr, status, mc_client;
41a524ab 7037 bool queue_thermal = false;
a59781bb
AD		 7038
7039 if (!rdev->ih.enabled || rdev->shutdown)
7040 return IRQ_NONE;
7041
7042 wptr = cik_get_ih_wptr(rdev);
7043
7044restart_ih:
7045 /* is somebody else already processing irqs? */
7046 if (atomic_xchg(&rdev->ih.lock, 1))
7047 return IRQ_NONE;
7048
7049 rptr = rdev->ih.rptr;
7050 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7051
7052 /* Order reading of wptr vs. reading of IH ring data */
7053 rmb();
7054
7055 /* display interrupts */
7056 cik_irq_ack(rdev);
7057
7058 while (rptr != wptr) {
7059 /* wptr/rptr are in bytes! */
7060 ring_index = rptr / 4;
7061 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7062 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7063 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7064
7065 switch (src_id) {
7066 case 1: /* D1 vblank/vline */
7067 switch (src_data) {
7068 case 0: /* D1 vblank */
7069 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7070 if (rdev->irq.crtc_vblank_int[0]) {
7071 drm_handle_vblank(rdev->ddev, 0);
7072 rdev->pm.vblank_sync = true;
7073 wake_up(&rdev->irq.vblank_queue);
7074 }
7075 if (atomic_read(&rdev->irq.pflip[0]))
7076 radeon_crtc_handle_flip(rdev, 0);
7077 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7078 DRM_DEBUG("IH: D1 vblank\n");
7079 }
7080 break;
7081 case 1: /* D1 vline */
7082 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7083 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7084 DRM_DEBUG("IH: D1 vline\n");
7085 }
7086 break;
7087 default:
7088 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7089 break;
7090 }
7091 break;
7092 case 2: /* D2 vblank/vline */
7093 switch (src_data) {
7094 case 0: /* D2 vblank */
7095 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7096 if (rdev->irq.crtc_vblank_int[1]) {
7097 drm_handle_vblank(rdev->ddev, 1);
7098 rdev->pm.vblank_sync = true;
7099 wake_up(&rdev->irq.vblank_queue);
7100 }
7101 if (atomic_read(&rdev->irq.pflip[1]))
7102 radeon_crtc_handle_flip(rdev, 1);
7103 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7104 DRM_DEBUG("IH: D2 vblank\n");
7105 }
7106 break;
7107 case 1: /* D2 vline */
7108 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7109 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7110 DRM_DEBUG("IH: D2 vline\n");
7111 }
7112 break;
7113 default:
7114 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7115 break;
7116 }
7117 break;
7118 case 3: /* D3 vblank/vline */
7119 switch (src_data) {
7120 case 0: /* D3 vblank */
7121 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7122 if (rdev->irq.crtc_vblank_int[2]) {
7123 drm_handle_vblank(rdev->ddev, 2);
7124 rdev->pm.vblank_sync = true;
7125 wake_up(&rdev->irq.vblank_queue);
7126 }
7127 if (atomic_read(&rdev->irq.pflip[2]))
7128 radeon_crtc_handle_flip(rdev, 2);
7129 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7130 DRM_DEBUG("IH: D3 vblank\n");
7131 }
7132 break;
7133 case 1: /* D3 vline */
7134 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7135 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7136 DRM_DEBUG("IH: D3 vline\n");
7137 }
7138 break;
7139 default:
7140 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7141 break;
7142 }
7143 break;
7144 case 4: /* D4 vblank/vline */
7145 switch (src_data) {
7146 case 0: /* D4 vblank */
7147 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7148 if (rdev->irq.crtc_vblank_int[3]) {
7149 drm_handle_vblank(rdev->ddev, 3);
7150 rdev->pm.vblank_sync = true;
7151 wake_up(&rdev->irq.vblank_queue);
7152 }
7153 if (atomic_read(&rdev->irq.pflip[3]))
7154 radeon_crtc_handle_flip(rdev, 3);
7155 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7156 DRM_DEBUG("IH: D4 vblank\n");
7157 }
7158 break;
7159 case 1: /* D4 vline */
7160 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7161 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7162 DRM_DEBUG("IH: D4 vline\n");
7163 }
7164 break;
7165 default:
7166 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7167 break;
7168 }
7169 break;
7170 case 5: /* D5 vblank/vline */
7171 switch (src_data) {
7172 case 0: /* D5 vblank */
7173 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7174 if (rdev->irq.crtc_vblank_int[4]) {
7175 drm_handle_vblank(rdev->ddev, 4);
7176 rdev->pm.vblank_sync = true;
7177 wake_up(&rdev->irq.vblank_queue);
7178 }
7179 if (atomic_read(&rdev->irq.pflip[4]))
7180 radeon_crtc_handle_flip(rdev, 4);
7181 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7182 DRM_DEBUG("IH: D5 vblank\n");
7183 }
7184 break;
7185 case 1: /* D5 vline */
7186 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7187 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7188 DRM_DEBUG("IH: D5 vline\n");
7189 }
7190 break;
7191 default:
7192 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7193 break;
7194 }
7195 break;
7196 case 6: /* D6 vblank/vline */
7197 switch (src_data) {
7198 case 0: /* D6 vblank */
7199 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7200 if (rdev->irq.crtc_vblank_int[5]) {
7201 drm_handle_vblank(rdev->ddev, 5);
7202 rdev->pm.vblank_sync = true;
7203 wake_up(&rdev->irq.vblank_queue);
7204 }
7205 if (atomic_read(&rdev->irq.pflip[5]))
7206 radeon_crtc_handle_flip(rdev, 5);
7207 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7208 DRM_DEBUG("IH: D6 vblank\n");
7209 }
7210 break;
7211 case 1: /* D6 vline */
7212 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7213 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7214 DRM_DEBUG("IH: D6 vline\n");
7215 }
7216 break;
7217 default:
7218 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7219 break;
7220 }
7221 break;
7222 case 42: /* HPD hotplug */
7223 switch (src_data) {
7224 case 0:
7225 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7226 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7227 queue_hotplug = true;
7228 DRM_DEBUG("IH: HPD1\n");
7229 }
7230 break;
7231 case 1:
7232 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7233 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7234 queue_hotplug = true;
7235 DRM_DEBUG("IH: HPD2\n");
7236 }
7237 break;
7238 case 2:
7239 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7240 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7241 queue_hotplug = true;
7242 DRM_DEBUG("IH: HPD3\n");
7243 }
7244 break;
7245 case 3:
7246 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7247 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7248 queue_hotplug = true;
7249 DRM_DEBUG("IH: HPD4\n");
7250 }
7251 break;
7252 case 4:
7253 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7254 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7255 queue_hotplug = true;
7256 DRM_DEBUG("IH: HPD5\n");
7257 }
7258 break;
7259 case 5:
7260 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7261 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7262 queue_hotplug = true;
7263 DRM_DEBUG("IH: HPD6\n");
7264 }
7265 break;
7266 default:
7267 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7268 break;
7269 }
7270 break;
6a3808b8
CK		 7271 case 124: /* UVD */
7272 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7273 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7274 break;
9d97c99b
AD		 7275 case 146:
7276 case 147:
3ec7d11b
AD		 7277 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7278 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7279 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9d97c99b
AD		 7280 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7281 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 7282 addr);
9d97c99b 7283 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 7284 status);
7285 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b
AD		 7286 /* reset addr and status */
7287 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7288 break;
a59781bb
AD		 7289 case 176: /* GFX RB CP_INT */
7290 case 177: /* GFX IB CP_INT */
7291 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7292 break;
7293 case 181: /* CP EOP event */
7294 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 7295 /* XXX check the bitfield order! */
7296 me_id = (ring_id & 0x60) >> 5;
7297 pipe_id = (ring_id & 0x18) >> 3;
7298 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7299 switch (me_id) {
7300 case 0:
7301 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7302 break;
7303 case 1:
a59781bb 7304 case 2:
2b0781a6
AD		 7305 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7306 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7307 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7308 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 7309 break;
7310 }
7311 break;
7312 case 184: /* CP Privileged reg access */
7313 DRM_ERROR("Illegal register access in command stream\n");
7314 /* XXX check the bitfield order! */
7315 me_id = (ring_id & 0x60) >> 5;
7316 pipe_id = (ring_id & 0x18) >> 3;
7317 queue_id = (ring_id & 0x7) >> 0;
7318 switch (me_id) {
7319 case 0:
7320 /* This results in a full GPU reset, but all we need to do is soft
7321 * reset the CP for gfx
7322 */
7323 queue_reset = true;
7324 break;
7325 case 1:
7326 /* XXX compute */
2b0781a6 7327 queue_reset = true;
a59781bb
AD		 7328 break;
7329 case 2:
7330 /* XXX compute */
2b0781a6 7331 queue_reset = true;
a59781bb
AD		 7332 break;
7333 }
7334 break;
7335 case 185: /* CP Privileged inst */
7336 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 7337 /* XXX check the bitfield order! */
7338 me_id = (ring_id & 0x60) >> 5;
7339 pipe_id = (ring_id & 0x18) >> 3;
7340 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7341 switch (me_id) {
7342 case 0:
7343 /* This results in a full GPU reset, but all we need to do is soft
7344 * reset the CP for gfx
7345 */
7346 queue_reset = true;
7347 break;
7348 case 1:
7349 /* XXX compute */
2b0781a6 7350 queue_reset = true;
a59781bb
AD		 7351 break;
7352 case 2:
7353 /* XXX compute */
2b0781a6 7354 queue_reset = true;
a59781bb
AD		 7355 break;
7356 }
7357 break;
21a93e13
AD		 7358 case 224: /* SDMA trap event */
7359 /* XXX check the bitfield order! */
7360 me_id = (ring_id & 0x3) >> 0;
7361 queue_id = (ring_id & 0xc) >> 2;
7362 DRM_DEBUG("IH: SDMA trap\n");
7363 switch (me_id) {
7364 case 0:
7365 switch (queue_id) {
7366 case 0:
7367 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7368 break;
7369 case 1:
7370 /* XXX compute */
7371 break;
7372 case 2:
7373 /* XXX compute */
7374 break;
7375 }
7376 break;
7377 case 1:
7378 switch (queue_id) {
7379 case 0:
7380 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7381 break;
7382 case 1:
7383 /* XXX compute */
7384 break;
7385 case 2:
7386 /* XXX compute */
7387 break;
7388 }
7389 break;
7390 }
7391 break;
41a524ab
AD		 7392 case 230: /* thermal low to high */
7393 DRM_DEBUG("IH: thermal low to high\n");
7394 rdev->pm.dpm.thermal.high_to_low = false;
7395 queue_thermal = true;
7396 break;
7397 case 231: /* thermal high to low */
7398 DRM_DEBUG("IH: thermal high to low\n");
7399 rdev->pm.dpm.thermal.high_to_low = true;
7400 queue_thermal = true;
7401 break;
7402 case 233: /* GUI IDLE */
7403 DRM_DEBUG("IH: GUI idle\n");
7404 break;
21a93e13
AD		 7405 case 241: /* SDMA Privileged inst */
7406 case 247: /* SDMA Privileged inst */
7407 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7408 /* XXX check the bitfield order! */
7409 me_id = (ring_id & 0x3) >> 0;
7410 queue_id = (ring_id & 0xc) >> 2;
7411 switch (me_id) {
7412 case 0:
7413 switch (queue_id) {
7414 case 0:
7415 queue_reset = true;
7416 break;
7417 case 1:
7418 /* XXX compute */
7419 queue_reset = true;
7420 break;
7421 case 2:
7422 /* XXX compute */
7423 queue_reset = true;
7424 break;
7425 }
7426 break;
7427 case 1:
7428 switch (queue_id) {
7429 case 0:
7430 queue_reset = true;
7431 break;
7432 case 1:
7433 /* XXX compute */
7434 queue_reset = true;
7435 break;
7436 case 2:
7437 /* XXX compute */
7438 queue_reset = true;
7439 break;
7440 }
7441 break;
7442 }
7443 break;
a59781bb
AD		 7444 default:
7445 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7446 break;
7447 }
7448
7449 /* wptr/rptr are in bytes! */
7450 rptr += 16;
7451 rptr &= rdev->ih.ptr_mask;
7452 }
7453 if (queue_hotplug)
7454 schedule_work(&rdev->hotplug_work);
7455 if (queue_reset)
7456 schedule_work(&rdev->reset_work);
41a524ab
AD		 7457 if (queue_thermal)
7458 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb
AD		 7459 rdev->ih.rptr = rptr;
7460 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7461 atomic_set(&rdev->ih.lock, 0);
7462
7463 /* make sure wptr hasn't changed while processing */
7464 wptr = cik_get_ih_wptr(rdev);
7465 if (wptr != rptr)
7466 goto restart_ih;
7467
7468 return IRQ_HANDLED;
7469}
7bf94a2c
AD		 7470
7471/*
7472 * startup/shutdown callbacks
7473 */
7474/**
7475 * cik_startup - program the asic to a functional state
7476 *
7477 * @rdev: radeon_device pointer
7478 *
7479 * Programs the asic to a functional state (CIK).
7480 * Called by cik_init() and cik_resume().
7481 * Returns 0 for success, error for failure.
7482 */
7483static int cik_startup(struct radeon_device *rdev)
7484{
7485 struct radeon_ring *ring;
7486 int r;
7487
8a7cd276
AD		 7488 /* enable pcie gen2/3 link */
7489 cik_pcie_gen3_enable(rdev);
7235711a
AD		 7490 /* enable aspm */
7491 cik_program_aspm(rdev);
8a7cd276 7492
e5903d39
AD		 7493 /* scratch needs to be initialized before MC */
7494 r = r600_vram_scratch_init(rdev);
7495 if (r)
7496 return r;
7497
6fab3feb
AD		 7498 cik_mc_program(rdev);
7499
7bf94a2c
AD		 7500 if (rdev->flags & RADEON_IS_IGP) {
7501 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7502 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
7503 r = cik_init_microcode(rdev);
7504 if (r) {
7505 DRM_ERROR("Failed to load firmware!\n");
7506 return r;
7507 }
7508 }
7509 } else {
7510 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7511 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7512 !rdev->mc_fw) {
7513 r = cik_init_microcode(rdev);
7514 if (r) {
7515 DRM_ERROR("Failed to load firmware!\n");
7516 return r;
7517 }
7518 }
7519
7520 r = ci_mc_load_microcode(rdev);
7521 if (r) {
7522 DRM_ERROR("Failed to load MC firmware!\n");
7523 return r;
7524 }
7525 }
7526
7bf94a2c
AD		 7527 r = cik_pcie_gart_enable(rdev);
7528 if (r)
7529 return r;
7530 cik_gpu_init(rdev);
7531
7532 /* allocate rlc buffers */
22c775ce
AD		 7533 if (rdev->flags & RADEON_IS_IGP) {
7534 if (rdev->family == CHIP_KAVERI) {
7535 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7536 rdev->rlc.reg_list_size =
7537 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7538 } else {
7539 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7540 rdev->rlc.reg_list_size =
7541 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7542 }
7543 }
7544 rdev->rlc.cs_data = ci_cs_data;
7545 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 7546 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 7547 if (r) {
7548 DRM_ERROR("Failed to init rlc BOs!\n");
7549 return r;
7550 }
7551
7552 /* allocate wb buffer */
7553 r = radeon_wb_init(rdev);
7554 if (r)
7555 return r;
7556
963e81f9
AD		 7557 /* allocate mec buffers */
7558 r = cik_mec_init(rdev);
7559 if (r) {
7560 DRM_ERROR("Failed to init MEC BOs!\n");
7561 return r;
7562 }
7563
7bf94a2c
AD		 7564 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7565 if (r) {
7566 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7567 return r;
7568 }
7569
963e81f9
AD		 7570 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7571 if (r) {
7572 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7573 return r;
7574 }
7575
7576 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7577 if (r) {
7578 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7579 return r;
7580 }
7581
7bf94a2c
AD		 7582 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7583 if (r) {
7584 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7585 return r;
7586 }
7587
7588 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7589 if (r) {
7590 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7591 return r;
7592 }
7593
2ce529da 7594 r = radeon_uvd_resume(rdev);
87167bb1 7595 if (!r) {
2ce529da
AD		 7596 r = uvd_v4_2_resume(rdev);
7597 if (!r) {
7598 r = radeon_fence_driver_start_ring(rdev,
7599 R600_RING_TYPE_UVD_INDEX);
7600 if (r)
7601 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7602 }
87167bb1
CK		 7603 }
7604 if (r)
7605 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7606
7bf94a2c
AD		 7607 /* Enable IRQ */
7608 if (!rdev->irq.installed) {
7609 r = radeon_irq_kms_init(rdev);
7610 if (r)
7611 return r;
7612 }
7613
7614 r = cik_irq_init(rdev);
7615 if (r) {
7616 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7617 radeon_irq_kms_fini(rdev);
7618 return r;
7619 }
7620 cik_irq_set(rdev);
7621
7622 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7623 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7624 CP_RB0_RPTR, CP_RB0_WPTR,
1dac28eb 7625 PACKET3(PACKET3_NOP, 0x3FFF));
7bf94a2c
AD		 7626 if (r)
7627 return r;
7628
963e81f9 7629 /* set up the compute queues */
2615b53a 7630 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7631 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7632 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7633 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
2e1e6dad 7634 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7635 if (r)
7636 return r;
7637 ring->me = 1; /* first MEC */
7638 ring->pipe = 0; /* first pipe */
7639 ring->queue = 0; /* first queue */
7640 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7641
2615b53a 7642 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7643 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7644 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7645 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
2e1e6dad 7646 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7647 if (r)
7648 return r;
7649 /* dGPU only have 1 MEC */
7650 ring->me = 1; /* first MEC */
7651 ring->pipe = 0; /* first pipe */
7652 ring->queue = 1; /* second queue */
7653 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7654
7bf94a2c
AD		 7655 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7656 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7657 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7658 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
2e1e6dad 7659 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7660 if (r)
7661 return r;
7662
7663 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7664 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7665 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7666 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
2e1e6dad 7667 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7668 if (r)
7669 return r;
7670
7671 r = cik_cp_resume(rdev);
7672 if (r)
7673 return r;
7674
7675 r = cik_sdma_resume(rdev);
7676 if (r)
7677 return r;
7678
87167bb1
CK		 7679 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7680 if (ring->ring_size) {
02c9f7fa 7681 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
87167bb1 7682 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2e1e6dad 7683 RADEON_CP_PACKET2);
87167bb1 7684 if (!r)
e409b128 7685 r = uvd_v1_0_init(rdev);
87167bb1
CK		 7686 if (r)
7687 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7688 }
7689
7bf94a2c
AD		 7690 r = radeon_ib_pool_init(rdev);
7691 if (r) {
7692 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7693 return r;
7694 }
7695
7696 r = radeon_vm_manager_init(rdev);
7697 if (r) {
7698 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7699 return r;
7700 }
7701
b530602f
AD		 7702 r = dce6_audio_init(rdev);
7703 if (r)
7704 return r;
7705
7bf94a2c
AD		 7706 return 0;
7707}
7708
7709/**
7710 * cik_resume - resume the asic to a functional state
7711 *
7712 * @rdev: radeon_device pointer
7713 *
7714 * Programs the asic to a functional state (CIK).
7715 * Called at resume.
7716 * Returns 0 for success, error for failure.
7717 */
7718int cik_resume(struct radeon_device *rdev)
7719{
7720 int r;
7721
7722 /* post card */
7723 atom_asic_init(rdev->mode_info.atom_context);
7724
0aafd313
AD		 7725 /* init golden registers */
7726 cik_init_golden_registers(rdev);
7727
7bf94a2c
AD		 7728 rdev->accel_working = true;
7729 r = cik_startup(rdev);
7730 if (r) {
7731 DRM_ERROR("cik startup failed on resume\n");
7732 rdev->accel_working = false;
7733 return r;
7734 }
7735
7736 return r;
7737
7738}
7739
7740/**
7741 * cik_suspend - suspend the asic
7742 *
7743 * @rdev: radeon_device pointer
7744 *
7745 * Bring the chip into a state suitable for suspend (CIK).
7746 * Called at suspend.
7747 * Returns 0 for success.
7748 */
7749int cik_suspend(struct radeon_device *rdev)
7750{
b530602f 7751 dce6_audio_fini(rdev);
7bf94a2c
AD		 7752 radeon_vm_manager_fini(rdev);
7753 cik_cp_enable(rdev, false);
7754 cik_sdma_enable(rdev, false);
e409b128 7755 uvd_v1_0_fini(rdev);
87167bb1 7756 radeon_uvd_suspend(rdev);
473359bc
AD		 7757 cik_fini_pg(rdev);
7758 cik_fini_cg(rdev);
7bf94a2c
AD		 7759 cik_irq_suspend(rdev);
7760 radeon_wb_disable(rdev);
7761 cik_pcie_gart_disable(rdev);
7762 return 0;
7763}
7764
7765/* Plan is to move initialization in that function and use
7766 * helper function so that radeon_device_init pretty much
7767 * do nothing more than calling asic specific function. This
7768 * should also allow to remove a bunch of callback function
7769 * like vram_info.
7770 */
7771/**
7772 * cik_init - asic specific driver and hw init
7773 *
7774 * @rdev: radeon_device pointer
7775 *
7776 * Setup asic specific driver variables and program the hw
7777 * to a functional state (CIK).
7778 * Called at driver startup.
7779 * Returns 0 for success, errors for failure.
7780 */
7781int cik_init(struct radeon_device *rdev)
7782{
7783 struct radeon_ring *ring;
7784 int r;
7785
7786 /* Read BIOS */
7787 if (!radeon_get_bios(rdev)) {
7788 if (ASIC_IS_AVIVO(rdev))
7789 return -EINVAL;
7790 }
7791 /* Must be an ATOMBIOS */
7792 if (!rdev->is_atom_bios) {
7793 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7794 return -EINVAL;
7795 }
7796 r = radeon_atombios_init(rdev);
7797 if (r)
7798 return r;
7799
7800 /* Post card if necessary */
7801 if (!radeon_card_posted(rdev)) {
7802 if (!rdev->bios) {
7803 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7804 return -EINVAL;
7805 }
7806 DRM_INFO("GPU not posted. posting now...\n");
7807 atom_asic_init(rdev->mode_info.atom_context);
7808 }
0aafd313
AD		 7809 /* init golden registers */
7810 cik_init_golden_registers(rdev);
7bf94a2c
AD		 7811 /* Initialize scratch registers */
7812 cik_scratch_init(rdev);
7813 /* Initialize surface registers */
7814 radeon_surface_init(rdev);
7815 /* Initialize clocks */
7816 radeon_get_clock_info(rdev->ddev);
7817
7818 /* Fence driver */
7819 r = radeon_fence_driver_init(rdev);
7820 if (r)
7821 return r;
7822
7823 /* initialize memory controller */
7824 r = cik_mc_init(rdev);
7825 if (r)
7826 return r;
7827 /* Memory manager */
7828 r = radeon_bo_init(rdev);
7829 if (r)
7830 return r;
7831
7832 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7833 ring->ring_obj = NULL;
7834 r600_ring_init(rdev, ring, 1024 * 1024);
7835
963e81f9
AD		 7836 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7837 ring->ring_obj = NULL;
7838 r600_ring_init(rdev, ring, 1024 * 1024);
7839 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7840 if (r)
7841 return r;
7842
7843 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7844 ring->ring_obj = NULL;
7845 r600_ring_init(rdev, ring, 1024 * 1024);
7846 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7847 if (r)
7848 return r;
7849
7bf94a2c
AD		 7850 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7851 ring->ring_obj = NULL;
7852 r600_ring_init(rdev, ring, 256 * 1024);
7853
7854 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7855 ring->ring_obj = NULL;
7856 r600_ring_init(rdev, ring, 256 * 1024);
7857
87167bb1
CK		 7858 r = radeon_uvd_init(rdev);
7859 if (!r) {
7860 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7861 ring->ring_obj = NULL;
7862 r600_ring_init(rdev, ring, 4096);
7863 }
7864
7bf94a2c
AD		 7865 rdev->ih.ring_obj = NULL;
7866 r600_ih_ring_init(rdev, 64 * 1024);
7867
7868 r = r600_pcie_gart_init(rdev);
7869 if (r)
7870 return r;
7871
7872 rdev->accel_working = true;
7873 r = cik_startup(rdev);
7874 if (r) {
7875 dev_err(rdev->dev, "disabling GPU acceleration\n");
7876 cik_cp_fini(rdev);
7877 cik_sdma_fini(rdev);
7878 cik_irq_fini(rdev);
1fd11777 7879 sumo_rlc_fini(rdev);
963e81f9 7880 cik_mec_fini(rdev);
7bf94a2c
AD		 7881 radeon_wb_fini(rdev);
7882 radeon_ib_pool_fini(rdev);
7883 radeon_vm_manager_fini(rdev);
7884 radeon_irq_kms_fini(rdev);
7885 cik_pcie_gart_fini(rdev);
7886 rdev->accel_working = false;
7887 }
7888
7889 /* Don't start up if the MC ucode is missing.
7890 * The default clocks and voltages before the MC ucode
7891 * is loaded are not suffient for advanced operations.
7892 */
7893 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7894 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7895 return -EINVAL;
7896 }
7897
7898 return 0;
7899}
7900
7901/**
7902 * cik_fini - asic specific driver and hw fini
7903 *
7904 * @rdev: radeon_device pointer
7905 *
7906 * Tear down the asic specific driver variables and program the hw
7907 * to an idle state (CIK).
7908 * Called at driver unload.
7909 */
7910void cik_fini(struct radeon_device *rdev)
7911{
7912 cik_cp_fini(rdev);
7913 cik_sdma_fini(rdev);
473359bc
AD		 7914 cik_fini_pg(rdev);
7915 cik_fini_cg(rdev);
7bf94a2c 7916 cik_irq_fini(rdev);
1fd11777 7917 sumo_rlc_fini(rdev);
963e81f9 7918 cik_mec_fini(rdev);
7bf94a2c
AD		 7919 radeon_wb_fini(rdev);
7920 radeon_vm_manager_fini(rdev);
7921 radeon_ib_pool_fini(rdev);
7922 radeon_irq_kms_fini(rdev);
e409b128 7923 uvd_v1_0_fini(rdev);
87167bb1 7924 radeon_uvd_fini(rdev);
7bf94a2c
AD		 7925 cik_pcie_gart_fini(rdev);
7926 r600_vram_scratch_fini(rdev);
7927 radeon_gem_fini(rdev);
7928 radeon_fence_driver_fini(rdev);
7929 radeon_bo_fini(rdev);
7930 radeon_atombios_fini(rdev);
7931 kfree(rdev->bios);
7932 rdev->bios = NULL;
7933}
cd84a27d 7934
134b480f
AD		 7935void dce8_program_fmt(struct drm_encoder *encoder)
7936{
7937 struct drm_device *dev = encoder->dev;
7938 struct radeon_device *rdev = dev->dev_private;
7939 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
7940 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
7941 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
7942 int bpc = 0;
7943 u32 tmp = 0;
6214bb74 7944 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 7945
6214bb74
AD		 7946 if (connector) {
7947 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 7948 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 7949 dither = radeon_connector->dither;
7950 }
134b480f
AD		 7951
7952 /* LVDS/eDP FMT is set up by atom */
7953 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
7954 return;
7955
7956 /* not needed for analog */
7957 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
7958 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
7959 return;
7960
7961 if (bpc == 0)
7962 return;
7963
7964 switch (bpc) {
7965 case 6:
6214bb74 7966 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7967 /* XXX sort out optimal dither settings */
7968 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7969 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
7970 else
7971 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
7972 break;
7973 case 8:
6214bb74 7974 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7975 /* XXX sort out optimal dither settings */
7976 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7977 FMT_RGB_RANDOM_ENABLE |
7978 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
7979 else
7980 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
7981 break;
7982 case 10:
6214bb74 7983 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7984 /* XXX sort out optimal dither settings */
7985 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7986 FMT_RGB_RANDOM_ENABLE |
7987 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
7988 else
7989 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
7990 break;
7991 default:
7992 /* not needed */
7993 break;
7994 }
7995
7996 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
7997}
7998
cd84a27d
AD		 7999/* display watermark setup */
8000/**
8001 * dce8_line_buffer_adjust - Set up the line buffer
8002 *
8003 * @rdev: radeon_device pointer
8004 * @radeon_crtc: the selected display controller
8005 * @mode: the current display mode on the selected display
8006 * controller
8007 *
8008 * Setup up the line buffer allocation for
8009 * the selected display controller (CIK).
8010 * Returns the line buffer size in pixels.
8011 */
8012static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8013 struct radeon_crtc *radeon_crtc,
8014 struct drm_display_mode *mode)
8015{
bc01a8c7
AD		 8016 u32 tmp, buffer_alloc, i;
8017 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8018 /*
8019 * Line Buffer Setup
8020 * There are 6 line buffers, one for each display controllers.
8021 * There are 3 partitions per LB. Select the number of partitions
8022 * to enable based on the display width. For display widths larger
8023 * than 4096, you need use to use 2 display controllers and combine
8024 * them using the stereo blender.
8025 */
8026 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 8027 if (mode->crtc_hdisplay < 1920) {
cd84a27d 8028 tmp = 1;
bc01a8c7
AD		 8029 buffer_alloc = 2;
8030 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 8031 tmp = 2;
bc01a8c7
AD		 8032 buffer_alloc = 2;
8033 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 8034 tmp = 0;
bc01a8c7
AD		 8035 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8036 } else {
cd84a27d
AD		 8037 DRM_DEBUG_KMS("Mode too big for LB!\n");
8038 tmp = 0;
bc01a8c7 8039 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 8040 }
bc01a8c7 8041 } else {
cd84a27d 8042 tmp = 1;
bc01a8c7
AD		 8043 buffer_alloc = 0;
8044 }
cd84a27d
AD		 8045
8046 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8047 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8048
bc01a8c7
AD		 8049 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8050 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8051 for (i = 0; i < rdev->usec_timeout; i++) {
8052 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8053 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8054 break;
8055 udelay(1);
8056 }
8057
cd84a27d
AD		 8058 if (radeon_crtc->base.enabled && mode) {
8059 switch (tmp) {
8060 case 0:
8061 default:
8062 return 4096 * 2;
8063 case 1:
8064 return 1920 * 2;
8065 case 2:
8066 return 2560 * 2;
8067 }
8068 }
8069
8070 /* controller not enabled, so no lb used */
8071 return 0;
8072}
8073
8074/**
8075 * cik_get_number_of_dram_channels - get the number of dram channels
8076 *
8077 * @rdev: radeon_device pointer
8078 *
8079 * Look up the number of video ram channels (CIK).
8080 * Used for display watermark bandwidth calculations
8081 * Returns the number of dram channels
8082 */
8083static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8084{
8085 u32 tmp = RREG32(MC_SHARED_CHMAP);
8086
8087 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8088 case 0:
8089 default:
8090 return 1;
8091 case 1:
8092 return 2;
8093 case 2:
8094 return 4;
8095 case 3:
8096 return 8;
8097 case 4:
8098 return 3;
8099 case 5:
8100 return 6;
8101 case 6:
8102 return 10;
8103 case 7:
8104 return 12;
8105 case 8:
8106 return 16;
8107 }
8108}
8109
8110struct dce8_wm_params {
8111 u32 dram_channels; /* number of dram channels */
8112 u32 yclk; /* bandwidth per dram data pin in kHz */
8113 u32 sclk; /* engine clock in kHz */
8114 u32 disp_clk; /* display clock in kHz */
8115 u32 src_width; /* viewport width */
8116 u32 active_time; /* active display time in ns */
8117 u32 blank_time; /* blank time in ns */
8118 bool interlaced; /* mode is interlaced */
8119 fixed20_12 vsc; /* vertical scale ratio */
8120 u32 num_heads; /* number of active crtcs */
8121 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8122 u32 lb_size; /* line buffer allocated to pipe */
8123 u32 vtaps; /* vertical scaler taps */
8124};
8125
8126/**
8127 * dce8_dram_bandwidth - get the dram bandwidth
8128 *
8129 * @wm: watermark calculation data
8130 *
8131 * Calculate the raw dram bandwidth (CIK).
8132 * Used for display watermark bandwidth calculations
8133 * Returns the dram bandwidth in MBytes/s
8134 */
8135static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8136{
8137 /* Calculate raw DRAM Bandwidth */
8138 fixed20_12 dram_efficiency; /* 0.7 */
8139 fixed20_12 yclk, dram_channels, bandwidth;
8140 fixed20_12 a;
8141
8142 a.full = dfixed_const(1000);
8143 yclk.full = dfixed_const(wm->yclk);
8144 yclk.full = dfixed_div(yclk, a);
8145 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8146 a.full = dfixed_const(10);
8147 dram_efficiency.full = dfixed_const(7);
8148 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8149 bandwidth.full = dfixed_mul(dram_channels, yclk);
8150 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8151
8152 return dfixed_trunc(bandwidth);
8153}
8154
8155/**
8156 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8157 *
8158 * @wm: watermark calculation data
8159 *
8160 * Calculate the dram bandwidth used for display (CIK).
8161 * Used for display watermark bandwidth calculations
8162 * Returns the dram bandwidth for display in MBytes/s
8163 */
8164static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8165{
8166 /* Calculate DRAM Bandwidth and the part allocated to display. */
8167 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8168 fixed20_12 yclk, dram_channels, bandwidth;
8169 fixed20_12 a;
8170
8171 a.full = dfixed_const(1000);
8172 yclk.full = dfixed_const(wm->yclk);
8173 yclk.full = dfixed_div(yclk, a);
8174 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8175 a.full = dfixed_const(10);
8176 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8177 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8178 bandwidth.full = dfixed_mul(dram_channels, yclk);
8179 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8180
8181 return dfixed_trunc(bandwidth);
8182}
8183
8184/**
8185 * dce8_data_return_bandwidth - get the data return bandwidth
8186 *
8187 * @wm: watermark calculation data
8188 *
8189 * Calculate the data return bandwidth used for display (CIK).
8190 * Used for display watermark bandwidth calculations
8191 * Returns the data return bandwidth in MBytes/s
8192 */
8193static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8194{
8195 /* Calculate the display Data return Bandwidth */
8196 fixed20_12 return_efficiency; /* 0.8 */
8197 fixed20_12 sclk, bandwidth;
8198 fixed20_12 a;
8199
8200 a.full = dfixed_const(1000);
8201 sclk.full = dfixed_const(wm->sclk);
8202 sclk.full = dfixed_div(sclk, a);
8203 a.full = dfixed_const(10);
8204 return_efficiency.full = dfixed_const(8);
8205 return_efficiency.full = dfixed_div(return_efficiency, a);
8206 a.full = dfixed_const(32);
8207 bandwidth.full = dfixed_mul(a, sclk);
8208 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8209
8210 return dfixed_trunc(bandwidth);
8211}
8212
8213/**
8214 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8215 *
8216 * @wm: watermark calculation data
8217 *
8218 * Calculate the dmif bandwidth used for display (CIK).
8219 * Used for display watermark bandwidth calculations
8220 * Returns the dmif bandwidth in MBytes/s
8221 */
8222static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8223{
8224 /* Calculate the DMIF Request Bandwidth */
8225 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8226 fixed20_12 disp_clk, bandwidth;
8227 fixed20_12 a, b;
8228
8229 a.full = dfixed_const(1000);
8230 disp_clk.full = dfixed_const(wm->disp_clk);
8231 disp_clk.full = dfixed_div(disp_clk, a);
8232 a.full = dfixed_const(32);
8233 b.full = dfixed_mul(a, disp_clk);
8234
8235 a.full = dfixed_const(10);
8236 disp_clk_request_efficiency.full = dfixed_const(8);
8237 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8238
8239 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8240
8241 return dfixed_trunc(bandwidth);
8242}
8243
8244/**
8245 * dce8_available_bandwidth - get the min available bandwidth
8246 *
8247 * @wm: watermark calculation data
8248 *
8249 * Calculate the min available bandwidth used for display (CIK).
8250 * Used for display watermark bandwidth calculations
8251 * Returns the min available bandwidth in MBytes/s
8252 */
8253static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8254{
8255 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8256 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8257 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8258 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8259
8260 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8261}
8262
8263/**
8264 * dce8_average_bandwidth - get the average available bandwidth
8265 *
8266 * @wm: watermark calculation data
8267 *
8268 * Calculate the average available bandwidth used for display (CIK).
8269 * Used for display watermark bandwidth calculations
8270 * Returns the average available bandwidth in MBytes/s
8271 */
8272static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8273{
8274 /* Calculate the display mode Average Bandwidth
8275 * DisplayMode should contain the source and destination dimensions,
8276 * timing, etc.
8277 */
8278 fixed20_12 bpp;
8279 fixed20_12 line_time;
8280 fixed20_12 src_width;
8281 fixed20_12 bandwidth;
8282 fixed20_12 a;
8283
8284 a.full = dfixed_const(1000);
8285 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8286 line_time.full = dfixed_div(line_time, a);
8287 bpp.full = dfixed_const(wm->bytes_per_pixel);
8288 src_width.full = dfixed_const(wm->src_width);
8289 bandwidth.full = dfixed_mul(src_width, bpp);
8290 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8291 bandwidth.full = dfixed_div(bandwidth, line_time);
8292
8293 return dfixed_trunc(bandwidth);
8294}
8295
8296/**
8297 * dce8_latency_watermark - get the latency watermark
8298 *
8299 * @wm: watermark calculation data
8300 *
8301 * Calculate the latency watermark (CIK).
8302 * Used for display watermark bandwidth calculations
8303 * Returns the latency watermark in ns
8304 */
8305static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8306{
8307 /* First calculate the latency in ns */
8308 u32 mc_latency = 2000; /* 2000 ns. */
8309 u32 available_bandwidth = dce8_available_bandwidth(wm);
8310 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8311 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8312 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8313 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8314 (wm->num_heads * cursor_line_pair_return_time);
8315 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8316 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8317 u32 tmp, dmif_size = 12288;
8318 fixed20_12 a, b, c;
8319
8320 if (wm->num_heads == 0)
8321 return 0;
8322
8323 a.full = dfixed_const(2);
8324 b.full = dfixed_const(1);
8325 if ((wm->vsc.full > a.full) ||
8326 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8327 (wm->vtaps >= 5) ||
8328 ((wm->vsc.full >= a.full) && wm->interlaced))
8329 max_src_lines_per_dst_line = 4;
8330 else
8331 max_src_lines_per_dst_line = 2;
8332
8333 a.full = dfixed_const(available_bandwidth);
8334 b.full = dfixed_const(wm->num_heads);
8335 a.full = dfixed_div(a, b);
8336
8337 b.full = dfixed_const(mc_latency + 512);
8338 c.full = dfixed_const(wm->disp_clk);
8339 b.full = dfixed_div(b, c);
8340
8341 c.full = dfixed_const(dmif_size);
8342 b.full = dfixed_div(c, b);
8343
8344 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8345
8346 b.full = dfixed_const(1000);
8347 c.full = dfixed_const(wm->disp_clk);
8348 b.full = dfixed_div(c, b);
8349 c.full = dfixed_const(wm->bytes_per_pixel);
8350 b.full = dfixed_mul(b, c);
8351
8352 lb_fill_bw = min(tmp, dfixed_trunc(b));
8353
8354 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8355 b.full = dfixed_const(1000);
8356 c.full = dfixed_const(lb_fill_bw);
8357 b.full = dfixed_div(c, b);
8358 a.full = dfixed_div(a, b);
8359 line_fill_time = dfixed_trunc(a);
8360
8361 if (line_fill_time < wm->active_time)
8362 return latency;
8363 else
8364 return latency + (line_fill_time - wm->active_time);
8365
8366}
8367
8368/**
8369 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8370 * average and available dram bandwidth
8371 *
8372 * @wm: watermark calculation data
8373 *
8374 * Check if the display average bandwidth fits in the display
8375 * dram bandwidth (CIK).
8376 * Used for display watermark bandwidth calculations
8377 * Returns true if the display fits, false if not.
8378 */
8379static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8380{
8381 if (dce8_average_bandwidth(wm) <=
8382 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8383 return true;
8384 else
8385 return false;
8386}
8387
8388/**
8389 * dce8_average_bandwidth_vs_available_bandwidth - check
8390 * average and available bandwidth
8391 *
8392 * @wm: watermark calculation data
8393 *
8394 * Check if the display average bandwidth fits in the display
8395 * available bandwidth (CIK).
8396 * Used for display watermark bandwidth calculations
8397 * Returns true if the display fits, false if not.
8398 */
8399static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8400{
8401 if (dce8_average_bandwidth(wm) <=
8402 (dce8_available_bandwidth(wm) / wm->num_heads))
8403 return true;
8404 else
8405 return false;
8406}
8407
8408/**
8409 * dce8_check_latency_hiding - check latency hiding
8410 *
8411 * @wm: watermark calculation data
8412 *
8413 * Check latency hiding (CIK).
8414 * Used for display watermark bandwidth calculations
8415 * Returns true if the display fits, false if not.
8416 */
8417static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8418{
8419 u32 lb_partitions = wm->lb_size / wm->src_width;
8420 u32 line_time = wm->active_time + wm->blank_time;
8421 u32 latency_tolerant_lines;
8422 u32 latency_hiding;
8423 fixed20_12 a;
8424
8425 a.full = dfixed_const(1);
8426 if (wm->vsc.full > a.full)
8427 latency_tolerant_lines = 1;
8428 else {
8429 if (lb_partitions <= (wm->vtaps + 1))
8430 latency_tolerant_lines = 1;
8431 else
8432 latency_tolerant_lines = 2;
8433 }
8434
8435 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8436
8437 if (dce8_latency_watermark(wm) <= latency_hiding)
8438 return true;
8439 else
8440 return false;
8441}
8442
8443/**
8444 * dce8_program_watermarks - program display watermarks
8445 *
8446 * @rdev: radeon_device pointer
8447 * @radeon_crtc: the selected display controller
8448 * @lb_size: line buffer size
8449 * @num_heads: number of display controllers in use
8450 *
8451 * Calculate and program the display watermarks for the
8452 * selected display controller (CIK).
8453 */
8454static void dce8_program_watermarks(struct radeon_device *rdev,
8455 struct radeon_crtc *radeon_crtc,
8456 u32 lb_size, u32 num_heads)
8457{
8458 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 8459 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 8460 u32 pixel_period;
8461 u32 line_time = 0;
8462 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8463 u32 tmp, wm_mask;
8464
8465 if (radeon_crtc->base.enabled && num_heads && mode) {
8466 pixel_period = 1000000 / (u32)mode->clock;
8467 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8468
58ea2dea
AD		 8469 /* watermark for high clocks */
8470 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8471 rdev->pm.dpm_enabled) {
8472 wm_high.yclk =
8473 radeon_dpm_get_mclk(rdev, false) * 10;
8474 wm_high.sclk =
8475 radeon_dpm_get_sclk(rdev, false) * 10;
8476 } else {
8477 wm_high.yclk = rdev->pm.current_mclk * 10;
8478 wm_high.sclk = rdev->pm.current_sclk * 10;
8479 }
8480
8481 wm_high.disp_clk = mode->clock;
8482 wm_high.src_width = mode->crtc_hdisplay;
8483 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8484 wm_high.blank_time = line_time - wm_high.active_time;
8485 wm_high.interlaced = false;
cd84a27d 8486 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 8487 wm_high.interlaced = true;
8488 wm_high.vsc = radeon_crtc->vsc;
8489 wm_high.vtaps = 1;
cd84a27d 8490 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 8491 wm_high.vtaps = 2;
8492 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8493 wm_high.lb_size = lb_size;
8494 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8495 wm_high.num_heads = num_heads;
cd84a27d
AD		 8496
8497 /* set for high clocks */
58ea2dea
AD		 8498 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8499
8500 /* possibly force display priority to high */
8501 /* should really do this at mode validation time... */
8502 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8503 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8504 !dce8_check_latency_hiding(&wm_high) ||
8505 (rdev->disp_priority == 2)) {
8506 DRM_DEBUG_KMS("force priority to high\n");
8507 }
8508
8509 /* watermark for low clocks */
8510 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8511 rdev->pm.dpm_enabled) {
8512 wm_low.yclk =
8513 radeon_dpm_get_mclk(rdev, true) * 10;
8514 wm_low.sclk =
8515 radeon_dpm_get_sclk(rdev, true) * 10;
8516 } else {
8517 wm_low.yclk = rdev->pm.current_mclk * 10;
8518 wm_low.sclk = rdev->pm.current_sclk * 10;
8519 }
8520
8521 wm_low.disp_clk = mode->clock;
8522 wm_low.src_width = mode->crtc_hdisplay;
8523 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8524 wm_low.blank_time = line_time - wm_low.active_time;
8525 wm_low.interlaced = false;
8526 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8527 wm_low.interlaced = true;
8528 wm_low.vsc = radeon_crtc->vsc;
8529 wm_low.vtaps = 1;
8530 if (radeon_crtc->rmx_type != RMX_OFF)
8531 wm_low.vtaps = 2;
8532 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8533 wm_low.lb_size = lb_size;
8534 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8535 wm_low.num_heads = num_heads;
8536
cd84a27d 8537 /* set for low clocks */
58ea2dea 8538 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 8539
8540 /* possibly force display priority to high */
8541 /* should really do this at mode validation time... */
58ea2dea
AD		 8542 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8543 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8544 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 8545 (rdev->disp_priority == 2)) {
8546 DRM_DEBUG_KMS("force priority to high\n");
8547 }
8548 }
8549
8550 /* select wm A */
8551 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8552 tmp = wm_mask;
8553 tmp &= ~LATENCY_WATERMARK_MASK(3);
8554 tmp |= LATENCY_WATERMARK_MASK(1);
8555 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8556 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8557 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8558 LATENCY_HIGH_WATERMARK(line_time)));
8559 /* select wm B */
8560 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8561 tmp &= ~LATENCY_WATERMARK_MASK(3);
8562 tmp |= LATENCY_WATERMARK_MASK(2);
8563 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8564 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8565 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8566 LATENCY_HIGH_WATERMARK(line_time)));
8567 /* restore original selection */
8568 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 8569
8570 /* save values for DPM */
8571 radeon_crtc->line_time = line_time;
8572 radeon_crtc->wm_high = latency_watermark_a;
8573 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 8574}
8575
8576/**
8577 * dce8_bandwidth_update - program display watermarks
8578 *
8579 * @rdev: radeon_device pointer
8580 *
8581 * Calculate and program the display watermarks and line
8582 * buffer allocation (CIK).
8583 */
8584void dce8_bandwidth_update(struct radeon_device *rdev)
8585{
8586 struct drm_display_mode *mode = NULL;
8587 u32 num_heads = 0, lb_size;
8588 int i;
8589
8590 radeon_update_display_priority(rdev);
8591
8592 for (i = 0; i < rdev->num_crtc; i++) {
8593 if (rdev->mode_info.crtcs[i]->base.enabled)
8594 num_heads++;
8595 }
8596 for (i = 0; i < rdev->num_crtc; i++) {
8597 mode = &rdev->mode_info.crtcs[i]->base.mode;
8598 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8599 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8600 }
8601}
44fa346f
AD		 8602
8603/**
8604 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8605 *
8606 * @rdev: radeon_device pointer
8607 *
8608 * Fetches a GPU clock counter snapshot (SI).
8609 * Returns the 64 bit clock counter snapshot.
8610 */
8611uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8612{
8613 uint64_t clock;
8614
8615 mutex_lock(&rdev->gpu_clock_mutex);
8616 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8617 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8618 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8619 mutex_unlock(&rdev->gpu_clock_mutex);
8620 return clock;
8621}
8622
87167bb1
CK		 8623static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8624 u32 cntl_reg, u32 status_reg)
8625{
8626 int r, i;
8627 struct atom_clock_dividers dividers;
8628 uint32_t tmp;
8629
8630 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8631 clock, false, &dividers);
8632 if (r)
8633 return r;
8634
8635 tmp = RREG32_SMC(cntl_reg);
8636 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8637 tmp |= dividers.post_divider;
8638 WREG32_SMC(cntl_reg, tmp);
8639
8640 for (i = 0; i < 100; i++) {
8641 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8642 break;
8643 mdelay(10);
8644 }
8645 if (i == 100)
8646 return -ETIMEDOUT;
8647
8648 return 0;
8649}
8650
8651int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8652{
8653 int r = 0;
8654
8655 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8656 if (r)
8657 return r;
8658
8659 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8660 return r;
8661}
8662
8a7cd276 8663static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 8664{
8a7cd276
AD		 8665 struct pci_dev *root = rdev->pdev->bus->self;
8666 int bridge_pos, gpu_pos;
8667 u32 speed_cntl, mask, current_data_rate;
8668 int ret, i;
8669 u16 tmp16;
87167bb1 8670
8a7cd276
AD		 8671 if (radeon_pcie_gen2 == 0)
8672 return;
87167bb1 8673
8a7cd276
AD		 8674 if (rdev->flags & RADEON_IS_IGP)
8675 return;
87167bb1 8676
8a7cd276
AD		 8677 if (!(rdev->flags & RADEON_IS_PCIE))
8678 return;
87167bb1 8679
8a7cd276
AD		 8680 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8681 if (ret != 0)
8682 return;
87167bb1 8683
8a7cd276
AD		 8684 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8685 return;
87167bb1 8686
8a7cd276
AD		 8687 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8688 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8689 LC_CURRENT_DATA_RATE_SHIFT;
8690 if (mask & DRM_PCIE_SPEED_80) {
8691 if (current_data_rate == 2) {
8692 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8693 return;
8694 }
8695 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8696 } else if (mask & DRM_PCIE_SPEED_50) {
8697 if (current_data_rate == 1) {
8698 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8699 return;
8700 }
8701 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8702 }
87167bb1 8703
8a7cd276
AD		 8704 bridge_pos = pci_pcie_cap(root);
8705 if (!bridge_pos)
8706 return;
8707
8708 gpu_pos = pci_pcie_cap(rdev->pdev);
8709 if (!gpu_pos)
8710 return;
8711
8712 if (mask & DRM_PCIE_SPEED_80) {
8713 /* re-try equalization if gen3 is not already enabled */
8714 if (current_data_rate != 2) {
8715 u16 bridge_cfg, gpu_cfg;
8716 u16 bridge_cfg2, gpu_cfg2;
8717 u32 max_lw, current_lw, tmp;
8718
8719 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8720 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8721
8722 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8723 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8724
8725 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8726 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8727
8728 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8729 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8730 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8731
8732 if (current_lw < max_lw) {
8733 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8734 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8735 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8736 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8737 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8738 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8739 }
8740 }
8741
8742 for (i = 0; i < 10; i++) {
8743 /* check status */
8744 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8745 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8746 break;
8747
8748 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8749 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8750
8751 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8752 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8753
8754 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8755 tmp |= LC_SET_QUIESCE;
8756 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8757
8758 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8759 tmp |= LC_REDO_EQ;
8760 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8761
8762 mdelay(100);
8763
8764 /* linkctl */
8765 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8766 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8767 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8768 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8769
8770 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8771 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8772 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8773 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8774
8775 /* linkctl2 */
8776 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8777 tmp16 &= ~((1 << 4) | (7 << 9));
8778 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8779 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8780
8781 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8782 tmp16 &= ~((1 << 4) | (7 << 9));
8783 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8784 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8785
8786 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8787 tmp &= ~LC_SET_QUIESCE;
8788 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8789 }
8790 }
8791 }
8792
8793 /* set the link speed */
8794 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8795 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8796 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8797
8798 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8799 tmp16 &= ~0xf;
8800 if (mask & DRM_PCIE_SPEED_80)
8801 tmp16 |= 3; /* gen3 */
8802 else if (mask & DRM_PCIE_SPEED_50)
8803 tmp16 |= 2; /* gen2 */
8804 else
8805 tmp16 |= 1; /* gen1 */
8806 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8807
8808 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8809 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8810 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8811
8812 for (i = 0; i < rdev->usec_timeout; i++) {
8813 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8814 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8815 break;
8816 udelay(1);
8817 }
8818}
7235711a
AD		 8819
8820static void cik_program_aspm(struct radeon_device *rdev)
8821{
8822 u32 data, orig;
8823 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8824 bool disable_clkreq = false;
8825
8826 if (radeon_aspm == 0)
8827 return;
8828
8829 /* XXX double check IGPs */
8830 if (rdev->flags & RADEON_IS_IGP)
8831 return;
8832
8833 if (!(rdev->flags & RADEON_IS_PCIE))
8834 return;
8835
8836 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8837 data &= ~LC_XMIT_N_FTS_MASK;
8838 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8839 if (orig != data)
8840 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8841
8842 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8843 data |= LC_GO_TO_RECOVERY;
8844 if (orig != data)
8845 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8846
8847 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8848 data |= P_IGNORE_EDB_ERR;
8849 if (orig != data)
8850 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8851
8852 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8853 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8854 data |= LC_PMI_TO_L1_DIS;
8855 if (!disable_l0s)
8856 data |= LC_L0S_INACTIVITY(7);
8857
8858 if (!disable_l1) {
8859 data |= LC_L1_INACTIVITY(7);
8860 data &= ~LC_PMI_TO_L1_DIS;
8861 if (orig != data)
8862 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8863
8864 if (!disable_plloff_in_l1) {
8865 bool clk_req_support;
8866
8867 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8868 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8869 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8870 if (orig != data)
8871 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8872
8873 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8874 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8875 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8876 if (orig != data)
8877 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8878
8879 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8880 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8881 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8882 if (orig != data)
8883 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8884
8885 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8886 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8887 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8888 if (orig != data)
8889 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8890
8891 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8892 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8893 data |= LC_DYN_LANES_PWR_STATE(3);
8894 if (orig != data)
8895 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8896
8897 if (!disable_clkreq) {
8898 struct pci_dev *root = rdev->pdev->bus->self;
8899 u32 lnkcap;
8900
8901 clk_req_support = false;
8902 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8903 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8904 clk_req_support = true;
8905 } else {
8906 clk_req_support = false;
8907 }
8908
8909 if (clk_req_support) {
8910 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8911 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8912 if (orig != data)
8913 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8914
8915 orig = data = RREG32_SMC(THM_CLK_CNTL);
8916 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8917 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8918 if (orig != data)
8919 WREG32_SMC(THM_CLK_CNTL, data);
8920
8921 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8922 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8923 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8924 if (orig != data)
8925 WREG32_SMC(MISC_CLK_CTRL, data);
8926
8927 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8928 data &= ~BCLK_AS_XCLK;
8929 if (orig != data)
8930 WREG32_SMC(CG_CLKPIN_CNTL, data);
8931
8932 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8933 data &= ~FORCE_BIF_REFCLK_EN;
8934 if (orig != data)
8935 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8936
8937 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8938 data &= ~MPLL_CLKOUT_SEL_MASK;
8939 data |= MPLL_CLKOUT_SEL(4);
8940 if (orig != data)
8941 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8942 }
8943 }
8944 } else {
8945 if (orig != data)
8946 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8947 }
8948
8949 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8950 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8951 if (orig != data)
8952 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8953
8954 if (!disable_l0s) {
8955 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8956 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8957 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8958 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8959 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8960 data &= ~LC_L0S_INACTIVITY_MASK;
8961 if (orig != data)
8962 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8963 }
8964 }
8965 }
87167bb1 8966}
Linux 6.x block layer and io_uring tree(s)