projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/dpm: retain user selected performance level across state changes
[linux-block.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
8cc1a532
AD		 30#include "cikd.h"
31#include "atom.h"
841cf442 32#include "cik_blit_shaders.h"
8c68e393 33#include "radeon_ucode.h"
22c775ce 34#include "clearstate_ci.h"
02c81327
AD		 35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
02c81327
AD		 44MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
45MODULE_FIRMWARE("radeon/KAVERI_me.bin");
46MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
47MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
48MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 49MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
02c81327
AD		 50MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
51MODULE_FIRMWARE("radeon/KABINI_me.bin");
52MODULE_FIRMWARE("radeon/KABINI_ce.bin");
53MODULE_FIRMWARE("radeon/KABINI_mec.bin");
54MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 55MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
02c81327 56
a59781bb
AD		 57extern int r600_ih_ring_alloc(struct radeon_device *rdev);
58extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 59extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
60extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 61extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 62extern void sumo_rlc_fini(struct radeon_device *rdev);
63extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 64extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 65extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 66extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
2483b4ea
CK		 67extern int cik_sdma_resume(struct radeon_device *rdev);
68extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
69extern void cik_sdma_fini(struct radeon_device *rdev);
70extern void cik_sdma_vm_set_page(struct radeon_device *rdev,
71 struct radeon_ib *ib,
72 uint64_t pe,
73 uint64_t addr, unsigned count,
74 uint32_t incr, uint32_t flags);
cc066715 75static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 76static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 77static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 78static void cik_init_pg(struct radeon_device *rdev);
79static void cik_init_cg(struct radeon_device *rdev);
4214faf6
AD		 80static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
81 bool enable);
6f2043ce 82
286d9cc6
AD		 83/* get temperature in millidegrees */
84int ci_get_temp(struct radeon_device *rdev)
85{
86 u32 temp;
87 int actual_temp = 0;
88
89 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
90 CTF_TEMP_SHIFT;
91
92 if (temp & 0x200)
93 actual_temp = 255;
94 else
95 actual_temp = temp & 0x1ff;
96
97 actual_temp = actual_temp * 1000;
98
99 return actual_temp;
100}
101
102/* get temperature in millidegrees */
103int kv_get_temp(struct radeon_device *rdev)
104{
105 u32 temp;
106 int actual_temp = 0;
107
108 temp = RREG32_SMC(0xC0300E0C);
109
110 if (temp)
111 actual_temp = (temp / 8) - 49;
112 else
113 actual_temp = 0;
114
115 actual_temp = actual_temp * 1000;
116
117 return actual_temp;
118}
6f2043ce 119
6e2c3c0a
AD		 120/*
121 * Indirect registers accessor
122 */
123u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
124{
0a5b7b0b 125 unsigned long flags;
6e2c3c0a
AD		 126 u32 r;
127
0a5b7b0b 128 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 129 WREG32(PCIE_INDEX, reg);
130 (void)RREG32(PCIE_INDEX);
131 r = RREG32(PCIE_DATA);
0a5b7b0b 132 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 133 return r;
134}
135
136void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
137{
0a5b7b0b
AD		 138 unsigned long flags;
139
140 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 141 WREG32(PCIE_INDEX, reg);
142 (void)RREG32(PCIE_INDEX);
143 WREG32(PCIE_DATA, v);
144 (void)RREG32(PCIE_DATA);
0a5b7b0b 145 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 146}
147
22c775ce
AD		 148static const u32 spectre_rlc_save_restore_register_list[] =
149{
150 (0x0e00 << 16) | (0xc12c >> 2),
151 0x00000000,
152 (0x0e00 << 16) | (0xc140 >> 2),
153 0x00000000,
154 (0x0e00 << 16) | (0xc150 >> 2),
155 0x00000000,
156 (0x0e00 << 16) | (0xc15c >> 2),
157 0x00000000,
158 (0x0e00 << 16) | (0xc168 >> 2),
159 0x00000000,
160 (0x0e00 << 16) | (0xc170 >> 2),
161 0x00000000,
162 (0x0e00 << 16) | (0xc178 >> 2),
163 0x00000000,
164 (0x0e00 << 16) | (0xc204 >> 2),
165 0x00000000,
166 (0x0e00 << 16) | (0xc2b4 >> 2),
167 0x00000000,
168 (0x0e00 << 16) | (0xc2b8 >> 2),
169 0x00000000,
170 (0x0e00 << 16) | (0xc2bc >> 2),
171 0x00000000,
172 (0x0e00 << 16) | (0xc2c0 >> 2),
173 0x00000000,
174 (0x0e00 << 16) | (0x8228 >> 2),
175 0x00000000,
176 (0x0e00 << 16) | (0x829c >> 2),
177 0x00000000,
178 (0x0e00 << 16) | (0x869c >> 2),
179 0x00000000,
180 (0x0600 << 16) | (0x98f4 >> 2),
181 0x00000000,
182 (0x0e00 << 16) | (0x98f8 >> 2),
183 0x00000000,
184 (0x0e00 << 16) | (0x9900 >> 2),
185 0x00000000,
186 (0x0e00 << 16) | (0xc260 >> 2),
187 0x00000000,
188 (0x0e00 << 16) | (0x90e8 >> 2),
189 0x00000000,
190 (0x0e00 << 16) | (0x3c000 >> 2),
191 0x00000000,
192 (0x0e00 << 16) | (0x3c00c >> 2),
193 0x00000000,
194 (0x0e00 << 16) | (0x8c1c >> 2),
195 0x00000000,
196 (0x0e00 << 16) | (0x9700 >> 2),
197 0x00000000,
198 (0x0e00 << 16) | (0xcd20 >> 2),
199 0x00000000,
200 (0x4e00 << 16) | (0xcd20 >> 2),
201 0x00000000,
202 (0x5e00 << 16) | (0xcd20 >> 2),
203 0x00000000,
204 (0x6e00 << 16) | (0xcd20 >> 2),
205 0x00000000,
206 (0x7e00 << 16) | (0xcd20 >> 2),
207 0x00000000,
208 (0x8e00 << 16) | (0xcd20 >> 2),
209 0x00000000,
210 (0x9e00 << 16) | (0xcd20 >> 2),
211 0x00000000,
212 (0xae00 << 16) | (0xcd20 >> 2),
213 0x00000000,
214 (0xbe00 << 16) | (0xcd20 >> 2),
215 0x00000000,
216 (0x0e00 << 16) | (0x89bc >> 2),
217 0x00000000,
218 (0x0e00 << 16) | (0x8900 >> 2),
219 0x00000000,
220 0x3,
221 (0x0e00 << 16) | (0xc130 >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0xc134 >> 2),
224 0x00000000,
225 (0x0e00 << 16) | (0xc1fc >> 2),
226 0x00000000,
227 (0x0e00 << 16) | (0xc208 >> 2),
228 0x00000000,
229 (0x0e00 << 16) | (0xc264 >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0xc268 >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0xc26c >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0xc270 >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0xc274 >> 2),
238 0x00000000,
239 (0x0e00 << 16) | (0xc278 >> 2),
240 0x00000000,
241 (0x0e00 << 16) | (0xc27c >> 2),
242 0x00000000,
243 (0x0e00 << 16) | (0xc280 >> 2),
244 0x00000000,
245 (0x0e00 << 16) | (0xc284 >> 2),
246 0x00000000,
247 (0x0e00 << 16) | (0xc288 >> 2),
248 0x00000000,
249 (0x0e00 << 16) | (0xc28c >> 2),
250 0x00000000,
251 (0x0e00 << 16) | (0xc290 >> 2),
252 0x00000000,
253 (0x0e00 << 16) | (0xc294 >> 2),
254 0x00000000,
255 (0x0e00 << 16) | (0xc298 >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0xc29c >> 2),
258 0x00000000,
259 (0x0e00 << 16) | (0xc2a0 >> 2),
260 0x00000000,
261 (0x0e00 << 16) | (0xc2a4 >> 2),
262 0x00000000,
263 (0x0e00 << 16) | (0xc2a8 >> 2),
264 0x00000000,
265 (0x0e00 << 16) | (0xc2ac >> 2),
266 0x00000000,
267 (0x0e00 << 16) | (0xc2b0 >> 2),
268 0x00000000,
269 (0x0e00 << 16) | (0x301d0 >> 2),
270 0x00000000,
271 (0x0e00 << 16) | (0x30238 >> 2),
272 0x00000000,
273 (0x0e00 << 16) | (0x30250 >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0x30254 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0x30258 >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0x3025c >> 2),
280 0x00000000,
281 (0x4e00 << 16) | (0xc900 >> 2),
282 0x00000000,
283 (0x5e00 << 16) | (0xc900 >> 2),
284 0x00000000,
285 (0x6e00 << 16) | (0xc900 >> 2),
286 0x00000000,
287 (0x7e00 << 16) | (0xc900 >> 2),
288 0x00000000,
289 (0x8e00 << 16) | (0xc900 >> 2),
290 0x00000000,
291 (0x9e00 << 16) | (0xc900 >> 2),
292 0x00000000,
293 (0xae00 << 16) | (0xc900 >> 2),
294 0x00000000,
295 (0xbe00 << 16) | (0xc900 >> 2),
296 0x00000000,
297 (0x4e00 << 16) | (0xc904 >> 2),
298 0x00000000,
299 (0x5e00 << 16) | (0xc904 >> 2),
300 0x00000000,
301 (0x6e00 << 16) | (0xc904 >> 2),
302 0x00000000,
303 (0x7e00 << 16) | (0xc904 >> 2),
304 0x00000000,
305 (0x8e00 << 16) | (0xc904 >> 2),
306 0x00000000,
307 (0x9e00 << 16) | (0xc904 >> 2),
308 0x00000000,
309 (0xae00 << 16) | (0xc904 >> 2),
310 0x00000000,
311 (0xbe00 << 16) | (0xc904 >> 2),
312 0x00000000,
313 (0x4e00 << 16) | (0xc908 >> 2),
314 0x00000000,
315 (0x5e00 << 16) | (0xc908 >> 2),
316 0x00000000,
317 (0x6e00 << 16) | (0xc908 >> 2),
318 0x00000000,
319 (0x7e00 << 16) | (0xc908 >> 2),
320 0x00000000,
321 (0x8e00 << 16) | (0xc908 >> 2),
322 0x00000000,
323 (0x9e00 << 16) | (0xc908 >> 2),
324 0x00000000,
325 (0xae00 << 16) | (0xc908 >> 2),
326 0x00000000,
327 (0xbe00 << 16) | (0xc908 >> 2),
328 0x00000000,
329 (0x4e00 << 16) | (0xc90c >> 2),
330 0x00000000,
331 (0x5e00 << 16) | (0xc90c >> 2),
332 0x00000000,
333 (0x6e00 << 16) | (0xc90c >> 2),
334 0x00000000,
335 (0x7e00 << 16) | (0xc90c >> 2),
336 0x00000000,
337 (0x8e00 << 16) | (0xc90c >> 2),
338 0x00000000,
339 (0x9e00 << 16) | (0xc90c >> 2),
340 0x00000000,
341 (0xae00 << 16) | (0xc90c >> 2),
342 0x00000000,
343 (0xbe00 << 16) | (0xc90c >> 2),
344 0x00000000,
345 (0x4e00 << 16) | (0xc910 >> 2),
346 0x00000000,
347 (0x5e00 << 16) | (0xc910 >> 2),
348 0x00000000,
349 (0x6e00 << 16) | (0xc910 >> 2),
350 0x00000000,
351 (0x7e00 << 16) | (0xc910 >> 2),
352 0x00000000,
353 (0x8e00 << 16) | (0xc910 >> 2),
354 0x00000000,
355 (0x9e00 << 16) | (0xc910 >> 2),
356 0x00000000,
357 (0xae00 << 16) | (0xc910 >> 2),
358 0x00000000,
359 (0xbe00 << 16) | (0xc910 >> 2),
360 0x00000000,
361 (0x0e00 << 16) | (0xc99c >> 2),
362 0x00000000,
363 (0x0e00 << 16) | (0x9834 >> 2),
364 0x00000000,
365 (0x0000 << 16) | (0x30f00 >> 2),
366 0x00000000,
367 (0x0001 << 16) | (0x30f00 >> 2),
368 0x00000000,
369 (0x0000 << 16) | (0x30f04 >> 2),
370 0x00000000,
371 (0x0001 << 16) | (0x30f04 >> 2),
372 0x00000000,
373 (0x0000 << 16) | (0x30f08 >> 2),
374 0x00000000,
375 (0x0001 << 16) | (0x30f08 >> 2),
376 0x00000000,
377 (0x0000 << 16) | (0x30f0c >> 2),
378 0x00000000,
379 (0x0001 << 16) | (0x30f0c >> 2),
380 0x00000000,
381 (0x0600 << 16) | (0x9b7c >> 2),
382 0x00000000,
383 (0x0e00 << 16) | (0x8a14 >> 2),
384 0x00000000,
385 (0x0e00 << 16) | (0x8a18 >> 2),
386 0x00000000,
387 (0x0600 << 16) | (0x30a00 >> 2),
388 0x00000000,
389 (0x0e00 << 16) | (0x8bf0 >> 2),
390 0x00000000,
391 (0x0e00 << 16) | (0x8bcc >> 2),
392 0x00000000,
393 (0x0e00 << 16) | (0x8b24 >> 2),
394 0x00000000,
395 (0x0e00 << 16) | (0x30a04 >> 2),
396 0x00000000,
397 (0x0600 << 16) | (0x30a10 >> 2),
398 0x00000000,
399 (0x0600 << 16) | (0x30a14 >> 2),
400 0x00000000,
401 (0x0600 << 16) | (0x30a18 >> 2),
402 0x00000000,
403 (0x0600 << 16) | (0x30a2c >> 2),
404 0x00000000,
405 (0x0e00 << 16) | (0xc700 >> 2),
406 0x00000000,
407 (0x0e00 << 16) | (0xc704 >> 2),
408 0x00000000,
409 (0x0e00 << 16) | (0xc708 >> 2),
410 0x00000000,
411 (0x0e00 << 16) | (0xc768 >> 2),
412 0x00000000,
413 (0x0400 << 16) | (0xc770 >> 2),
414 0x00000000,
415 (0x0400 << 16) | (0xc774 >> 2),
416 0x00000000,
417 (0x0400 << 16) | (0xc778 >> 2),
418 0x00000000,
419 (0x0400 << 16) | (0xc77c >> 2),
420 0x00000000,
421 (0x0400 << 16) | (0xc780 >> 2),
422 0x00000000,
423 (0x0400 << 16) | (0xc784 >> 2),
424 0x00000000,
425 (0x0400 << 16) | (0xc788 >> 2),
426 0x00000000,
427 (0x0400 << 16) | (0xc78c >> 2),
428 0x00000000,
429 (0x0400 << 16) | (0xc798 >> 2),
430 0x00000000,
431 (0x0400 << 16) | (0xc79c >> 2),
432 0x00000000,
433 (0x0400 << 16) | (0xc7a0 >> 2),
434 0x00000000,
435 (0x0400 << 16) | (0xc7a4 >> 2),
436 0x00000000,
437 (0x0400 << 16) | (0xc7a8 >> 2),
438 0x00000000,
439 (0x0400 << 16) | (0xc7ac >> 2),
440 0x00000000,
441 (0x0400 << 16) | (0xc7b0 >> 2),
442 0x00000000,
443 (0x0400 << 16) | (0xc7b4 >> 2),
444 0x00000000,
445 (0x0e00 << 16) | (0x9100 >> 2),
446 0x00000000,
447 (0x0e00 << 16) | (0x3c010 >> 2),
448 0x00000000,
449 (0x0e00 << 16) | (0x92a8 >> 2),
450 0x00000000,
451 (0x0e00 << 16) | (0x92ac >> 2),
452 0x00000000,
453 (0x0e00 << 16) | (0x92b4 >> 2),
454 0x00000000,
455 (0x0e00 << 16) | (0x92b8 >> 2),
456 0x00000000,
457 (0x0e00 << 16) | (0x92bc >> 2),
458 0x00000000,
459 (0x0e00 << 16) | (0x92c0 >> 2),
460 0x00000000,
461 (0x0e00 << 16) | (0x92c4 >> 2),
462 0x00000000,
463 (0x0e00 << 16) | (0x92c8 >> 2),
464 0x00000000,
465 (0x0e00 << 16) | (0x92cc >> 2),
466 0x00000000,
467 (0x0e00 << 16) | (0x92d0 >> 2),
468 0x00000000,
469 (0x0e00 << 16) | (0x8c00 >> 2),
470 0x00000000,
471 (0x0e00 << 16) | (0x8c04 >> 2),
472 0x00000000,
473 (0x0e00 << 16) | (0x8c20 >> 2),
474 0x00000000,
475 (0x0e00 << 16) | (0x8c38 >> 2),
476 0x00000000,
477 (0x0e00 << 16) | (0x8c3c >> 2),
478 0x00000000,
479 (0x0e00 << 16) | (0xae00 >> 2),
480 0x00000000,
481 (0x0e00 << 16) | (0x9604 >> 2),
482 0x00000000,
483 (0x0e00 << 16) | (0xac08 >> 2),
484 0x00000000,
485 (0x0e00 << 16) | (0xac0c >> 2),
486 0x00000000,
487 (0x0e00 << 16) | (0xac10 >> 2),
488 0x00000000,
489 (0x0e00 << 16) | (0xac14 >> 2),
490 0x00000000,
491 (0x0e00 << 16) | (0xac58 >> 2),
492 0x00000000,
493 (0x0e00 << 16) | (0xac68 >> 2),
494 0x00000000,
495 (0x0e00 << 16) | (0xac6c >> 2),
496 0x00000000,
497 (0x0e00 << 16) | (0xac70 >> 2),
498 0x00000000,
499 (0x0e00 << 16) | (0xac74 >> 2),
500 0x00000000,
501 (0x0e00 << 16) | (0xac78 >> 2),
502 0x00000000,
503 (0x0e00 << 16) | (0xac7c >> 2),
504 0x00000000,
505 (0x0e00 << 16) | (0xac80 >> 2),
506 0x00000000,
507 (0x0e00 << 16) | (0xac84 >> 2),
508 0x00000000,
509 (0x0e00 << 16) | (0xac88 >> 2),
510 0x00000000,
511 (0x0e00 << 16) | (0xac8c >> 2),
512 0x00000000,
513 (0x0e00 << 16) | (0x970c >> 2),
514 0x00000000,
515 (0x0e00 << 16) | (0x9714 >> 2),
516 0x00000000,
517 (0x0e00 << 16) | (0x9718 >> 2),
518 0x00000000,
519 (0x0e00 << 16) | (0x971c >> 2),
520 0x00000000,
521 (0x0e00 << 16) | (0x31068 >> 2),
522 0x00000000,
523 (0x4e00 << 16) | (0x31068 >> 2),
524 0x00000000,
525 (0x5e00 << 16) | (0x31068 >> 2),
526 0x00000000,
527 (0x6e00 << 16) | (0x31068 >> 2),
528 0x00000000,
529 (0x7e00 << 16) | (0x31068 >> 2),
530 0x00000000,
531 (0x8e00 << 16) | (0x31068 >> 2),
532 0x00000000,
533 (0x9e00 << 16) | (0x31068 >> 2),
534 0x00000000,
535 (0xae00 << 16) | (0x31068 >> 2),
536 0x00000000,
537 (0xbe00 << 16) | (0x31068 >> 2),
538 0x00000000,
539 (0x0e00 << 16) | (0xcd10 >> 2),
540 0x00000000,
541 (0x0e00 << 16) | (0xcd14 >> 2),
542 0x00000000,
543 (0x0e00 << 16) | (0x88b0 >> 2),
544 0x00000000,
545 (0x0e00 << 16) | (0x88b4 >> 2),
546 0x00000000,
547 (0x0e00 << 16) | (0x88b8 >> 2),
548 0x00000000,
549 (0x0e00 << 16) | (0x88bc >> 2),
550 0x00000000,
551 (0x0400 << 16) | (0x89c0 >> 2),
552 0x00000000,
553 (0x0e00 << 16) | (0x88c4 >> 2),
554 0x00000000,
555 (0x0e00 << 16) | (0x88c8 >> 2),
556 0x00000000,
557 (0x0e00 << 16) | (0x88d0 >> 2),
558 0x00000000,
559 (0x0e00 << 16) | (0x88d4 >> 2),
560 0x00000000,
561 (0x0e00 << 16) | (0x88d8 >> 2),
562 0x00000000,
563 (0x0e00 << 16) | (0x8980 >> 2),
564 0x00000000,
565 (0x0e00 << 16) | (0x30938 >> 2),
566 0x00000000,
567 (0x0e00 << 16) | (0x3093c >> 2),
568 0x00000000,
569 (0x0e00 << 16) | (0x30940 >> 2),
570 0x00000000,
571 (0x0e00 << 16) | (0x89a0 >> 2),
572 0x00000000,
573 (0x0e00 << 16) | (0x30900 >> 2),
574 0x00000000,
575 (0x0e00 << 16) | (0x30904 >> 2),
576 0x00000000,
577 (0x0e00 << 16) | (0x89b4 >> 2),
578 0x00000000,
579 (0x0e00 << 16) | (0x3c210 >> 2),
580 0x00000000,
581 (0x0e00 << 16) | (0x3c214 >> 2),
582 0x00000000,
583 (0x0e00 << 16) | (0x3c218 >> 2),
584 0x00000000,
585 (0x0e00 << 16) | (0x8904 >> 2),
586 0x00000000,
587 0x5,
588 (0x0e00 << 16) | (0x8c28 >> 2),
589 (0x0e00 << 16) | (0x8c2c >> 2),
590 (0x0e00 << 16) | (0x8c30 >> 2),
591 (0x0e00 << 16) | (0x8c34 >> 2),
592 (0x0e00 << 16) | (0x9600 >> 2),
593};
594
595static const u32 kalindi_rlc_save_restore_register_list[] =
596{
597 (0x0e00 << 16) | (0xc12c >> 2),
598 0x00000000,
599 (0x0e00 << 16) | (0xc140 >> 2),
600 0x00000000,
601 (0x0e00 << 16) | (0xc150 >> 2),
602 0x00000000,
603 (0x0e00 << 16) | (0xc15c >> 2),
604 0x00000000,
605 (0x0e00 << 16) | (0xc168 >> 2),
606 0x00000000,
607 (0x0e00 << 16) | (0xc170 >> 2),
608 0x00000000,
609 (0x0e00 << 16) | (0xc204 >> 2),
610 0x00000000,
611 (0x0e00 << 16) | (0xc2b4 >> 2),
612 0x00000000,
613 (0x0e00 << 16) | (0xc2b8 >> 2),
614 0x00000000,
615 (0x0e00 << 16) | (0xc2bc >> 2),
616 0x00000000,
617 (0x0e00 << 16) | (0xc2c0 >> 2),
618 0x00000000,
619 (0x0e00 << 16) | (0x8228 >> 2),
620 0x00000000,
621 (0x0e00 << 16) | (0x829c >> 2),
622 0x00000000,
623 (0x0e00 << 16) | (0x869c >> 2),
624 0x00000000,
625 (0x0600 << 16) | (0x98f4 >> 2),
626 0x00000000,
627 (0x0e00 << 16) | (0x98f8 >> 2),
628 0x00000000,
629 (0x0e00 << 16) | (0x9900 >> 2),
630 0x00000000,
631 (0x0e00 << 16) | (0xc260 >> 2),
632 0x00000000,
633 (0x0e00 << 16) | (0x90e8 >> 2),
634 0x00000000,
635 (0x0e00 << 16) | (0x3c000 >> 2),
636 0x00000000,
637 (0x0e00 << 16) | (0x3c00c >> 2),
638 0x00000000,
639 (0x0e00 << 16) | (0x8c1c >> 2),
640 0x00000000,
641 (0x0e00 << 16) | (0x9700 >> 2),
642 0x00000000,
643 (0x0e00 << 16) | (0xcd20 >> 2),
644 0x00000000,
645 (0x4e00 << 16) | (0xcd20 >> 2),
646 0x00000000,
647 (0x5e00 << 16) | (0xcd20 >> 2),
648 0x00000000,
649 (0x6e00 << 16) | (0xcd20 >> 2),
650 0x00000000,
651 (0x7e00 << 16) | (0xcd20 >> 2),
652 0x00000000,
653 (0x0e00 << 16) | (0x89bc >> 2),
654 0x00000000,
655 (0x0e00 << 16) | (0x8900 >> 2),
656 0x00000000,
657 0x3,
658 (0x0e00 << 16) | (0xc130 >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xc134 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xc1fc >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0xc208 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0xc264 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0xc268 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0xc26c >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc270 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0xc274 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0xc28c >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0xc290 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0xc294 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0xc298 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0xc2a0 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0xc2a4 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0xc2a8 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0xc2ac >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x301d0 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x30238 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x30250 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x30254 >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x30258 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x3025c >> 2),
703 0x00000000,
704 (0x4e00 << 16) | (0xc900 >> 2),
705 0x00000000,
706 (0x5e00 << 16) | (0xc900 >> 2),
707 0x00000000,
708 (0x6e00 << 16) | (0xc900 >> 2),
709 0x00000000,
710 (0x7e00 << 16) | (0xc900 >> 2),
711 0x00000000,
712 (0x4e00 << 16) | (0xc904 >> 2),
713 0x00000000,
714 (0x5e00 << 16) | (0xc904 >> 2),
715 0x00000000,
716 (0x6e00 << 16) | (0xc904 >> 2),
717 0x00000000,
718 (0x7e00 << 16) | (0xc904 >> 2),
719 0x00000000,
720 (0x4e00 << 16) | (0xc908 >> 2),
721 0x00000000,
722 (0x5e00 << 16) | (0xc908 >> 2),
723 0x00000000,
724 (0x6e00 << 16) | (0xc908 >> 2),
725 0x00000000,
726 (0x7e00 << 16) | (0xc908 >> 2),
727 0x00000000,
728 (0x4e00 << 16) | (0xc90c >> 2),
729 0x00000000,
730 (0x5e00 << 16) | (0xc90c >> 2),
731 0x00000000,
732 (0x6e00 << 16) | (0xc90c >> 2),
733 0x00000000,
734 (0x7e00 << 16) | (0xc90c >> 2),
735 0x00000000,
736 (0x4e00 << 16) | (0xc910 >> 2),
737 0x00000000,
738 (0x5e00 << 16) | (0xc910 >> 2),
739 0x00000000,
740 (0x6e00 << 16) | (0xc910 >> 2),
741 0x00000000,
742 (0x7e00 << 16) | (0xc910 >> 2),
743 0x00000000,
744 (0x0e00 << 16) | (0xc99c >> 2),
745 0x00000000,
746 (0x0e00 << 16) | (0x9834 >> 2),
747 0x00000000,
748 (0x0000 << 16) | (0x30f00 >> 2),
749 0x00000000,
750 (0x0000 << 16) | (0x30f04 >> 2),
751 0x00000000,
752 (0x0000 << 16) | (0x30f08 >> 2),
753 0x00000000,
754 (0x0000 << 16) | (0x30f0c >> 2),
755 0x00000000,
756 (0x0600 << 16) | (0x9b7c >> 2),
757 0x00000000,
758 (0x0e00 << 16) | (0x8a14 >> 2),
759 0x00000000,
760 (0x0e00 << 16) | (0x8a18 >> 2),
761 0x00000000,
762 (0x0600 << 16) | (0x30a00 >> 2),
763 0x00000000,
764 (0x0e00 << 16) | (0x8bf0 >> 2),
765 0x00000000,
766 (0x0e00 << 16) | (0x8bcc >> 2),
767 0x00000000,
768 (0x0e00 << 16) | (0x8b24 >> 2),
769 0x00000000,
770 (0x0e00 << 16) | (0x30a04 >> 2),
771 0x00000000,
772 (0x0600 << 16) | (0x30a10 >> 2),
773 0x00000000,
774 (0x0600 << 16) | (0x30a14 >> 2),
775 0x00000000,
776 (0x0600 << 16) | (0x30a18 >> 2),
777 0x00000000,
778 (0x0600 << 16) | (0x30a2c >> 2),
779 0x00000000,
780 (0x0e00 << 16) | (0xc700 >> 2),
781 0x00000000,
782 (0x0e00 << 16) | (0xc704 >> 2),
783 0x00000000,
784 (0x0e00 << 16) | (0xc708 >> 2),
785 0x00000000,
786 (0x0e00 << 16) | (0xc768 >> 2),
787 0x00000000,
788 (0x0400 << 16) | (0xc770 >> 2),
789 0x00000000,
790 (0x0400 << 16) | (0xc774 >> 2),
791 0x00000000,
792 (0x0400 << 16) | (0xc798 >> 2),
793 0x00000000,
794 (0x0400 << 16) | (0xc79c >> 2),
795 0x00000000,
796 (0x0e00 << 16) | (0x9100 >> 2),
797 0x00000000,
798 (0x0e00 << 16) | (0x3c010 >> 2),
799 0x00000000,
800 (0x0e00 << 16) | (0x8c00 >> 2),
801 0x00000000,
802 (0x0e00 << 16) | (0x8c04 >> 2),
803 0x00000000,
804 (0x0e00 << 16) | (0x8c20 >> 2),
805 0x00000000,
806 (0x0e00 << 16) | (0x8c38 >> 2),
807 0x00000000,
808 (0x0e00 << 16) | (0x8c3c >> 2),
809 0x00000000,
810 (0x0e00 << 16) | (0xae00 >> 2),
811 0x00000000,
812 (0x0e00 << 16) | (0x9604 >> 2),
813 0x00000000,
814 (0x0e00 << 16) | (0xac08 >> 2),
815 0x00000000,
816 (0x0e00 << 16) | (0xac0c >> 2),
817 0x00000000,
818 (0x0e00 << 16) | (0xac10 >> 2),
819 0x00000000,
820 (0x0e00 << 16) | (0xac14 >> 2),
821 0x00000000,
822 (0x0e00 << 16) | (0xac58 >> 2),
823 0x00000000,
824 (0x0e00 << 16) | (0xac68 >> 2),
825 0x00000000,
826 (0x0e00 << 16) | (0xac6c >> 2),
827 0x00000000,
828 (0x0e00 << 16) | (0xac70 >> 2),
829 0x00000000,
830 (0x0e00 << 16) | (0xac74 >> 2),
831 0x00000000,
832 (0x0e00 << 16) | (0xac78 >> 2),
833 0x00000000,
834 (0x0e00 << 16) | (0xac7c >> 2),
835 0x00000000,
836 (0x0e00 << 16) | (0xac80 >> 2),
837 0x00000000,
838 (0x0e00 << 16) | (0xac84 >> 2),
839 0x00000000,
840 (0x0e00 << 16) | (0xac88 >> 2),
841 0x00000000,
842 (0x0e00 << 16) | (0xac8c >> 2),
843 0x00000000,
844 (0x0e00 << 16) | (0x970c >> 2),
845 0x00000000,
846 (0x0e00 << 16) | (0x9714 >> 2),
847 0x00000000,
848 (0x0e00 << 16) | (0x9718 >> 2),
849 0x00000000,
850 (0x0e00 << 16) | (0x971c >> 2),
851 0x00000000,
852 (0x0e00 << 16) | (0x31068 >> 2),
853 0x00000000,
854 (0x4e00 << 16) | (0x31068 >> 2),
855 0x00000000,
856 (0x5e00 << 16) | (0x31068 >> 2),
857 0x00000000,
858 (0x6e00 << 16) | (0x31068 >> 2),
859 0x00000000,
860 (0x7e00 << 16) | (0x31068 >> 2),
861 0x00000000,
862 (0x0e00 << 16) | (0xcd10 >> 2),
863 0x00000000,
864 (0x0e00 << 16) | (0xcd14 >> 2),
865 0x00000000,
866 (0x0e00 << 16) | (0x88b0 >> 2),
867 0x00000000,
868 (0x0e00 << 16) | (0x88b4 >> 2),
869 0x00000000,
870 (0x0e00 << 16) | (0x88b8 >> 2),
871 0x00000000,
872 (0x0e00 << 16) | (0x88bc >> 2),
873 0x00000000,
874 (0x0400 << 16) | (0x89c0 >> 2),
875 0x00000000,
876 (0x0e00 << 16) | (0x88c4 >> 2),
877 0x00000000,
878 (0x0e00 << 16) | (0x88c8 >> 2),
879 0x00000000,
880 (0x0e00 << 16) | (0x88d0 >> 2),
881 0x00000000,
882 (0x0e00 << 16) | (0x88d4 >> 2),
883 0x00000000,
884 (0x0e00 << 16) | (0x88d8 >> 2),
885 0x00000000,
886 (0x0e00 << 16) | (0x8980 >> 2),
887 0x00000000,
888 (0x0e00 << 16) | (0x30938 >> 2),
889 0x00000000,
890 (0x0e00 << 16) | (0x3093c >> 2),
891 0x00000000,
892 (0x0e00 << 16) | (0x30940 >> 2),
893 0x00000000,
894 (0x0e00 << 16) | (0x89a0 >> 2),
895 0x00000000,
896 (0x0e00 << 16) | (0x30900 >> 2),
897 0x00000000,
898 (0x0e00 << 16) | (0x30904 >> 2),
899 0x00000000,
900 (0x0e00 << 16) | (0x89b4 >> 2),
901 0x00000000,
902 (0x0e00 << 16) | (0x3e1fc >> 2),
903 0x00000000,
904 (0x0e00 << 16) | (0x3c210 >> 2),
905 0x00000000,
906 (0x0e00 << 16) | (0x3c214 >> 2),
907 0x00000000,
908 (0x0e00 << 16) | (0x3c218 >> 2),
909 0x00000000,
910 (0x0e00 << 16) | (0x8904 >> 2),
911 0x00000000,
912 0x5,
913 (0x0e00 << 16) | (0x8c28 >> 2),
914 (0x0e00 << 16) | (0x8c2c >> 2),
915 (0x0e00 << 16) | (0x8c30 >> 2),
916 (0x0e00 << 16) | (0x8c34 >> 2),
917 (0x0e00 << 16) | (0x9600 >> 2),
918};
919
0aafd313
AD		 920static const u32 bonaire_golden_spm_registers[] =
921{
922 0x30800, 0xe0ffffff, 0xe0000000
923};
924
925static const u32 bonaire_golden_common_registers[] =
926{
927 0xc770, 0xffffffff, 0x00000800,
928 0xc774, 0xffffffff, 0x00000800,
929 0xc798, 0xffffffff, 0x00007fbf,
930 0xc79c, 0xffffffff, 0x00007faf
931};
932
933static const u32 bonaire_golden_registers[] =
934{
935 0x3354, 0x00000333, 0x00000333,
936 0x3350, 0x000c0fc0, 0x00040200,
937 0x9a10, 0x00010000, 0x00058208,
938 0x3c000, 0xffff1fff, 0x00140000,
939 0x3c200, 0xfdfc0fff, 0x00000100,
940 0x3c234, 0x40000000, 0x40000200,
941 0x9830, 0xffffffff, 0x00000000,
942 0x9834, 0xf00fffff, 0x00000400,
943 0x9838, 0x0002021c, 0x00020200,
944 0xc78, 0x00000080, 0x00000000,
945 0x5bb0, 0x000000f0, 0x00000070,
946 0x5bc0, 0xf0311fff, 0x80300000,
947 0x98f8, 0x73773777, 0x12010001,
948 0x350c, 0x00810000, 0x408af000,
949 0x7030, 0x31000111, 0x00000011,
950 0x2f48, 0x73773777, 0x12010001,
951 0x220c, 0x00007fb6, 0x0021a1b1,
952 0x2210, 0x00007fb6, 0x002021b1,
953 0x2180, 0x00007fb6, 0x00002191,
954 0x2218, 0x00007fb6, 0x002121b1,
955 0x221c, 0x00007fb6, 0x002021b1,
956 0x21dc, 0x00007fb6, 0x00002191,
957 0x21e0, 0x00007fb6, 0x00002191,
958 0x3628, 0x0000003f, 0x0000000a,
959 0x362c, 0x0000003f, 0x0000000a,
960 0x2ae4, 0x00073ffe, 0x000022a2,
961 0x240c, 0x000007ff, 0x00000000,
962 0x8a14, 0xf000003f, 0x00000007,
963 0x8bf0, 0x00002001, 0x00000001,
964 0x8b24, 0xffffffff, 0x00ffffff,
965 0x30a04, 0x0000ff0f, 0x00000000,
966 0x28a4c, 0x07ffffff, 0x06000000,
967 0x4d8, 0x00000fff, 0x00000100,
968 0x3e78, 0x00000001, 0x00000002,
969 0x9100, 0x03000000, 0x0362c688,
970 0x8c00, 0x000000ff, 0x00000001,
971 0xe40, 0x00001fff, 0x00001fff,
972 0x9060, 0x0000007f, 0x00000020,
973 0x9508, 0x00010000, 0x00010000,
974 0xac14, 0x000003ff, 0x000000f3,
975 0xac0c, 0xffffffff, 0x00001032
976};
977
978static const u32 bonaire_mgcg_cgcg_init[] =
979{
980 0xc420, 0xffffffff, 0xfffffffc,
981 0x30800, 0xffffffff, 0xe0000000,
982 0x3c2a0, 0xffffffff, 0x00000100,
983 0x3c208, 0xffffffff, 0x00000100,
984 0x3c2c0, 0xffffffff, 0xc0000100,
985 0x3c2c8, 0xffffffff, 0xc0000100,
986 0x3c2c4, 0xffffffff, 0xc0000100,
987 0x55e4, 0xffffffff, 0x00600100,
988 0x3c280, 0xffffffff, 0x00000100,
989 0x3c214, 0xffffffff, 0x06000100,
990 0x3c220, 0xffffffff, 0x00000100,
991 0x3c218, 0xffffffff, 0x06000100,
992 0x3c204, 0xffffffff, 0x00000100,
993 0x3c2e0, 0xffffffff, 0x00000100,
994 0x3c224, 0xffffffff, 0x00000100,
995 0x3c200, 0xffffffff, 0x00000100,
996 0x3c230, 0xffffffff, 0x00000100,
997 0x3c234, 0xffffffff, 0x00000100,
998 0x3c250, 0xffffffff, 0x00000100,
999 0x3c254, 0xffffffff, 0x00000100,
1000 0x3c258, 0xffffffff, 0x00000100,
1001 0x3c25c, 0xffffffff, 0x00000100,
1002 0x3c260, 0xffffffff, 0x00000100,
1003 0x3c27c, 0xffffffff, 0x00000100,
1004 0x3c278, 0xffffffff, 0x00000100,
1005 0x3c210, 0xffffffff, 0x06000100,
1006 0x3c290, 0xffffffff, 0x00000100,
1007 0x3c274, 0xffffffff, 0x00000100,
1008 0x3c2b4, 0xffffffff, 0x00000100,
1009 0x3c2b0, 0xffffffff, 0x00000100,
1010 0x3c270, 0xffffffff, 0x00000100,
1011 0x30800, 0xffffffff, 0xe0000000,
1012 0x3c020, 0xffffffff, 0x00010000,
1013 0x3c024, 0xffffffff, 0x00030002,
1014 0x3c028, 0xffffffff, 0x00040007,
1015 0x3c02c, 0xffffffff, 0x00060005,
1016 0x3c030, 0xffffffff, 0x00090008,
1017 0x3c034, 0xffffffff, 0x00010000,
1018 0x3c038, 0xffffffff, 0x00030002,
1019 0x3c03c, 0xffffffff, 0x00040007,
1020 0x3c040, 0xffffffff, 0x00060005,
1021 0x3c044, 0xffffffff, 0x00090008,
1022 0x3c048, 0xffffffff, 0x00010000,
1023 0x3c04c, 0xffffffff, 0x00030002,
1024 0x3c050, 0xffffffff, 0x00040007,
1025 0x3c054, 0xffffffff, 0x00060005,
1026 0x3c058, 0xffffffff, 0x00090008,
1027 0x3c05c, 0xffffffff, 0x00010000,
1028 0x3c060, 0xffffffff, 0x00030002,
1029 0x3c064, 0xffffffff, 0x00040007,
1030 0x3c068, 0xffffffff, 0x00060005,
1031 0x3c06c, 0xffffffff, 0x00090008,
1032 0x3c070, 0xffffffff, 0x00010000,
1033 0x3c074, 0xffffffff, 0x00030002,
1034 0x3c078, 0xffffffff, 0x00040007,
1035 0x3c07c, 0xffffffff, 0x00060005,
1036 0x3c080, 0xffffffff, 0x00090008,
1037 0x3c084, 0xffffffff, 0x00010000,
1038 0x3c088, 0xffffffff, 0x00030002,
1039 0x3c08c, 0xffffffff, 0x00040007,
1040 0x3c090, 0xffffffff, 0x00060005,
1041 0x3c094, 0xffffffff, 0x00090008,
1042 0x3c098, 0xffffffff, 0x00010000,
1043 0x3c09c, 0xffffffff, 0x00030002,
1044 0x3c0a0, 0xffffffff, 0x00040007,
1045 0x3c0a4, 0xffffffff, 0x00060005,
1046 0x3c0a8, 0xffffffff, 0x00090008,
1047 0x3c000, 0xffffffff, 0x96e00200,
1048 0x8708, 0xffffffff, 0x00900100,
1049 0xc424, 0xffffffff, 0x0020003f,
1050 0x38, 0xffffffff, 0x0140001c,
1051 0x3c, 0x000f0000, 0x000f0000,
1052 0x220, 0xffffffff, 0xC060000C,
1053 0x224, 0xc0000fff, 0x00000100,
1054 0xf90, 0xffffffff, 0x00000100,
1055 0xf98, 0x00000101, 0x00000000,
1056 0x20a8, 0xffffffff, 0x00000104,
1057 0x55e4, 0xff000fff, 0x00000100,
1058 0x30cc, 0xc0000fff, 0x00000104,
1059 0xc1e4, 0x00000001, 0x00000001,
1060 0xd00c, 0xff000ff0, 0x00000100,
1061 0xd80c, 0xff000ff0, 0x00000100
1062};
1063
1064static const u32 spectre_golden_spm_registers[] =
1065{
1066 0x30800, 0xe0ffffff, 0xe0000000
1067};
1068
1069static const u32 spectre_golden_common_registers[] =
1070{
1071 0xc770, 0xffffffff, 0x00000800,
1072 0xc774, 0xffffffff, 0x00000800,
1073 0xc798, 0xffffffff, 0x00007fbf,
1074 0xc79c, 0xffffffff, 0x00007faf
1075};
1076
1077static const u32 spectre_golden_registers[] =
1078{
1079 0x3c000, 0xffff1fff, 0x96940200,
1080 0x3c00c, 0xffff0001, 0xff000000,
1081 0x3c200, 0xfffc0fff, 0x00000100,
1082 0x6ed8, 0x00010101, 0x00010000,
1083 0x9834, 0xf00fffff, 0x00000400,
1084 0x9838, 0xfffffffc, 0x00020200,
1085 0x5bb0, 0x000000f0, 0x00000070,
1086 0x5bc0, 0xf0311fff, 0x80300000,
1087 0x98f8, 0x73773777, 0x12010001,
1088 0x9b7c, 0x00ff0000, 0x00fc0000,
1089 0x2f48, 0x73773777, 0x12010001,
1090 0x8a14, 0xf000003f, 0x00000007,
1091 0x8b24, 0xffffffff, 0x00ffffff,
1092 0x28350, 0x3f3f3fff, 0x00000082,
1093 0x28355, 0x0000003f, 0x00000000,
1094 0x3e78, 0x00000001, 0x00000002,
1095 0x913c, 0xffff03df, 0x00000004,
1096 0xc768, 0x00000008, 0x00000008,
1097 0x8c00, 0x000008ff, 0x00000800,
1098 0x9508, 0x00010000, 0x00010000,
1099 0xac0c, 0xffffffff, 0x54763210,
1100 0x214f8, 0x01ff01ff, 0x00000002,
1101 0x21498, 0x007ff800, 0x00200000,
1102 0x2015c, 0xffffffff, 0x00000f40,
1103 0x30934, 0xffffffff, 0x00000001
1104};
1105
1106static const u32 spectre_mgcg_cgcg_init[] =
1107{
1108 0xc420, 0xffffffff, 0xfffffffc,
1109 0x30800, 0xffffffff, 0xe0000000,
1110 0x3c2a0, 0xffffffff, 0x00000100,
1111 0x3c208, 0xffffffff, 0x00000100,
1112 0x3c2c0, 0xffffffff, 0x00000100,
1113 0x3c2c8, 0xffffffff, 0x00000100,
1114 0x3c2c4, 0xffffffff, 0x00000100,
1115 0x55e4, 0xffffffff, 0x00600100,
1116 0x3c280, 0xffffffff, 0x00000100,
1117 0x3c214, 0xffffffff, 0x06000100,
1118 0x3c220, 0xffffffff, 0x00000100,
1119 0x3c218, 0xffffffff, 0x06000100,
1120 0x3c204, 0xffffffff, 0x00000100,
1121 0x3c2e0, 0xffffffff, 0x00000100,
1122 0x3c224, 0xffffffff, 0x00000100,
1123 0x3c200, 0xffffffff, 0x00000100,
1124 0x3c230, 0xffffffff, 0x00000100,
1125 0x3c234, 0xffffffff, 0x00000100,
1126 0x3c250, 0xffffffff, 0x00000100,
1127 0x3c254, 0xffffffff, 0x00000100,
1128 0x3c258, 0xffffffff, 0x00000100,
1129 0x3c25c, 0xffffffff, 0x00000100,
1130 0x3c260, 0xffffffff, 0x00000100,
1131 0x3c27c, 0xffffffff, 0x00000100,
1132 0x3c278, 0xffffffff, 0x00000100,
1133 0x3c210, 0xffffffff, 0x06000100,
1134 0x3c290, 0xffffffff, 0x00000100,
1135 0x3c274, 0xffffffff, 0x00000100,
1136 0x3c2b4, 0xffffffff, 0x00000100,
1137 0x3c2b0, 0xffffffff, 0x00000100,
1138 0x3c270, 0xffffffff, 0x00000100,
1139 0x30800, 0xffffffff, 0xe0000000,
1140 0x3c020, 0xffffffff, 0x00010000,
1141 0x3c024, 0xffffffff, 0x00030002,
1142 0x3c028, 0xffffffff, 0x00040007,
1143 0x3c02c, 0xffffffff, 0x00060005,
1144 0x3c030, 0xffffffff, 0x00090008,
1145 0x3c034, 0xffffffff, 0x00010000,
1146 0x3c038, 0xffffffff, 0x00030002,
1147 0x3c03c, 0xffffffff, 0x00040007,
1148 0x3c040, 0xffffffff, 0x00060005,
1149 0x3c044, 0xffffffff, 0x00090008,
1150 0x3c048, 0xffffffff, 0x00010000,
1151 0x3c04c, 0xffffffff, 0x00030002,
1152 0x3c050, 0xffffffff, 0x00040007,
1153 0x3c054, 0xffffffff, 0x00060005,
1154 0x3c058, 0xffffffff, 0x00090008,
1155 0x3c05c, 0xffffffff, 0x00010000,
1156 0x3c060, 0xffffffff, 0x00030002,
1157 0x3c064, 0xffffffff, 0x00040007,
1158 0x3c068, 0xffffffff, 0x00060005,
1159 0x3c06c, 0xffffffff, 0x00090008,
1160 0x3c070, 0xffffffff, 0x00010000,
1161 0x3c074, 0xffffffff, 0x00030002,
1162 0x3c078, 0xffffffff, 0x00040007,
1163 0x3c07c, 0xffffffff, 0x00060005,
1164 0x3c080, 0xffffffff, 0x00090008,
1165 0x3c084, 0xffffffff, 0x00010000,
1166 0x3c088, 0xffffffff, 0x00030002,
1167 0x3c08c, 0xffffffff, 0x00040007,
1168 0x3c090, 0xffffffff, 0x00060005,
1169 0x3c094, 0xffffffff, 0x00090008,
1170 0x3c098, 0xffffffff, 0x00010000,
1171 0x3c09c, 0xffffffff, 0x00030002,
1172 0x3c0a0, 0xffffffff, 0x00040007,
1173 0x3c0a4, 0xffffffff, 0x00060005,
1174 0x3c0a8, 0xffffffff, 0x00090008,
1175 0x3c0ac, 0xffffffff, 0x00010000,
1176 0x3c0b0, 0xffffffff, 0x00030002,
1177 0x3c0b4, 0xffffffff, 0x00040007,
1178 0x3c0b8, 0xffffffff, 0x00060005,
1179 0x3c0bc, 0xffffffff, 0x00090008,
1180 0x3c000, 0xffffffff, 0x96e00200,
1181 0x8708, 0xffffffff, 0x00900100,
1182 0xc424, 0xffffffff, 0x0020003f,
1183 0x38, 0xffffffff, 0x0140001c,
1184 0x3c, 0x000f0000, 0x000f0000,
1185 0x220, 0xffffffff, 0xC060000C,
1186 0x224, 0xc0000fff, 0x00000100,
1187 0xf90, 0xffffffff, 0x00000100,
1188 0xf98, 0x00000101, 0x00000000,
1189 0x20a8, 0xffffffff, 0x00000104,
1190 0x55e4, 0xff000fff, 0x00000100,
1191 0x30cc, 0xc0000fff, 0x00000104,
1192 0xc1e4, 0x00000001, 0x00000001,
1193 0xd00c, 0xff000ff0, 0x00000100,
1194 0xd80c, 0xff000ff0, 0x00000100
1195};
1196
1197static const u32 kalindi_golden_spm_registers[] =
1198{
1199 0x30800, 0xe0ffffff, 0xe0000000
1200};
1201
1202static const u32 kalindi_golden_common_registers[] =
1203{
1204 0xc770, 0xffffffff, 0x00000800,
1205 0xc774, 0xffffffff, 0x00000800,
1206 0xc798, 0xffffffff, 0x00007fbf,
1207 0xc79c, 0xffffffff, 0x00007faf
1208};
1209
1210static const u32 kalindi_golden_registers[] =
1211{
1212 0x3c000, 0xffffdfff, 0x6e944040,
1213 0x55e4, 0xff607fff, 0xfc000100,
1214 0x3c220, 0xff000fff, 0x00000100,
1215 0x3c224, 0xff000fff, 0x00000100,
1216 0x3c200, 0xfffc0fff, 0x00000100,
1217 0x6ed8, 0x00010101, 0x00010000,
1218 0x9830, 0xffffffff, 0x00000000,
1219 0x9834, 0xf00fffff, 0x00000400,
1220 0x5bb0, 0x000000f0, 0x00000070,
1221 0x5bc0, 0xf0311fff, 0x80300000,
1222 0x98f8, 0x73773777, 0x12010001,
1223 0x98fc, 0xffffffff, 0x00000010,
1224 0x9b7c, 0x00ff0000, 0x00fc0000,
1225 0x8030, 0x00001f0f, 0x0000100a,
1226 0x2f48, 0x73773777, 0x12010001,
1227 0x2408, 0x000fffff, 0x000c007f,
1228 0x8a14, 0xf000003f, 0x00000007,
1229 0x8b24, 0x3fff3fff, 0x00ffcfff,
1230 0x30a04, 0x0000ff0f, 0x00000000,
1231 0x28a4c, 0x07ffffff, 0x06000000,
1232 0x4d8, 0x00000fff, 0x00000100,
1233 0x3e78, 0x00000001, 0x00000002,
1234 0xc768, 0x00000008, 0x00000008,
1235 0x8c00, 0x000000ff, 0x00000003,
1236 0x214f8, 0x01ff01ff, 0x00000002,
1237 0x21498, 0x007ff800, 0x00200000,
1238 0x2015c, 0xffffffff, 0x00000f40,
1239 0x88c4, 0x001f3ae3, 0x00000082,
1240 0x88d4, 0x0000001f, 0x00000010,
1241 0x30934, 0xffffffff, 0x00000000
1242};
1243
1244static const u32 kalindi_mgcg_cgcg_init[] =
1245{
1246 0xc420, 0xffffffff, 0xfffffffc,
1247 0x30800, 0xffffffff, 0xe0000000,
1248 0x3c2a0, 0xffffffff, 0x00000100,
1249 0x3c208, 0xffffffff, 0x00000100,
1250 0x3c2c0, 0xffffffff, 0x00000100,
1251 0x3c2c8, 0xffffffff, 0x00000100,
1252 0x3c2c4, 0xffffffff, 0x00000100,
1253 0x55e4, 0xffffffff, 0x00600100,
1254 0x3c280, 0xffffffff, 0x00000100,
1255 0x3c214, 0xffffffff, 0x06000100,
1256 0x3c220, 0xffffffff, 0x00000100,
1257 0x3c218, 0xffffffff, 0x06000100,
1258 0x3c204, 0xffffffff, 0x00000100,
1259 0x3c2e0, 0xffffffff, 0x00000100,
1260 0x3c224, 0xffffffff, 0x00000100,
1261 0x3c200, 0xffffffff, 0x00000100,
1262 0x3c230, 0xffffffff, 0x00000100,
1263 0x3c234, 0xffffffff, 0x00000100,
1264 0x3c250, 0xffffffff, 0x00000100,
1265 0x3c254, 0xffffffff, 0x00000100,
1266 0x3c258, 0xffffffff, 0x00000100,
1267 0x3c25c, 0xffffffff, 0x00000100,
1268 0x3c260, 0xffffffff, 0x00000100,
1269 0x3c27c, 0xffffffff, 0x00000100,
1270 0x3c278, 0xffffffff, 0x00000100,
1271 0x3c210, 0xffffffff, 0x06000100,
1272 0x3c290, 0xffffffff, 0x00000100,
1273 0x3c274, 0xffffffff, 0x00000100,
1274 0x3c2b4, 0xffffffff, 0x00000100,
1275 0x3c2b0, 0xffffffff, 0x00000100,
1276 0x3c270, 0xffffffff, 0x00000100,
1277 0x30800, 0xffffffff, 0xe0000000,
1278 0x3c020, 0xffffffff, 0x00010000,
1279 0x3c024, 0xffffffff, 0x00030002,
1280 0x3c028, 0xffffffff, 0x00040007,
1281 0x3c02c, 0xffffffff, 0x00060005,
1282 0x3c030, 0xffffffff, 0x00090008,
1283 0x3c034, 0xffffffff, 0x00010000,
1284 0x3c038, 0xffffffff, 0x00030002,
1285 0x3c03c, 0xffffffff, 0x00040007,
1286 0x3c040, 0xffffffff, 0x00060005,
1287 0x3c044, 0xffffffff, 0x00090008,
1288 0x3c000, 0xffffffff, 0x96e00200,
1289 0x8708, 0xffffffff, 0x00900100,
1290 0xc424, 0xffffffff, 0x0020003f,
1291 0x38, 0xffffffff, 0x0140001c,
1292 0x3c, 0x000f0000, 0x000f0000,
1293 0x220, 0xffffffff, 0xC060000C,
1294 0x224, 0xc0000fff, 0x00000100,
1295 0x20a8, 0xffffffff, 0x00000104,
1296 0x55e4, 0xff000fff, 0x00000100,
1297 0x30cc, 0xc0000fff, 0x00000104,
1298 0xc1e4, 0x00000001, 0x00000001,
1299 0xd00c, 0xff000ff0, 0x00000100,
1300 0xd80c, 0xff000ff0, 0x00000100
1301};
1302
1303static void cik_init_golden_registers(struct radeon_device *rdev)
1304{
1305 switch (rdev->family) {
1306 case CHIP_BONAIRE:
1307 radeon_program_register_sequence(rdev,
1308 bonaire_mgcg_cgcg_init,
1309 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1310 radeon_program_register_sequence(rdev,
1311 bonaire_golden_registers,
1312 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1313 radeon_program_register_sequence(rdev,
1314 bonaire_golden_common_registers,
1315 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1316 radeon_program_register_sequence(rdev,
1317 bonaire_golden_spm_registers,
1318 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1319 break;
1320 case CHIP_KABINI:
1321 radeon_program_register_sequence(rdev,
1322 kalindi_mgcg_cgcg_init,
1323 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1324 radeon_program_register_sequence(rdev,
1325 kalindi_golden_registers,
1326 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1327 radeon_program_register_sequence(rdev,
1328 kalindi_golden_common_registers,
1329 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1330 radeon_program_register_sequence(rdev,
1331 kalindi_golden_spm_registers,
1332 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1333 break;
1334 case CHIP_KAVERI:
1335 radeon_program_register_sequence(rdev,
1336 spectre_mgcg_cgcg_init,
1337 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1338 radeon_program_register_sequence(rdev,
1339 spectre_golden_registers,
1340 (const u32)ARRAY_SIZE(spectre_golden_registers));
1341 radeon_program_register_sequence(rdev,
1342 spectre_golden_common_registers,
1343 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1344 radeon_program_register_sequence(rdev,
1345 spectre_golden_spm_registers,
1346 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1347 break;
1348 default:
1349 break;
1350 }
1351}
1352
2c67912c
AD		 1353/**
1354 * cik_get_xclk - get the xclk
1355 *
1356 * @rdev: radeon_device pointer
1357 *
1358 * Returns the reference clock used by the gfx engine
1359 * (CIK).
1360 */
1361u32 cik_get_xclk(struct radeon_device *rdev)
1362{
1363 u32 reference_clock = rdev->clock.spll.reference_freq;
1364
1365 if (rdev->flags & RADEON_IS_IGP) {
1366 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1367 return reference_clock / 2;
1368 } else {
1369 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1370 return reference_clock / 4;
1371 }
1372 return reference_clock;
1373}
1374
75efdee1
AD		 1375/**
1376 * cik_mm_rdoorbell - read a doorbell dword
1377 *
1378 * @rdev: radeon_device pointer
1379 * @offset: byte offset into the aperture
1380 *
1381 * Returns the value in the doorbell aperture at the
1382 * requested offset (CIK).
1383 */
1384u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
1385{
1386 if (offset < rdev->doorbell.size) {
1387 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
1388 } else {
1389 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
1390 return 0;
1391 }
1392}
1393
1394/**
1395 * cik_mm_wdoorbell - write a doorbell dword
1396 *
1397 * @rdev: radeon_device pointer
1398 * @offset: byte offset into the aperture
1399 * @v: value to write
1400 *
1401 * Writes @v to the doorbell aperture at the
1402 * requested offset (CIK).
1403 */
1404void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
1405{
1406 if (offset < rdev->doorbell.size) {
1407 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
1408 } else {
1409 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
1410 }
1411}
1412
bc8273fe
AD		 1413#define BONAIRE_IO_MC_REGS_SIZE 36
1414
1415static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1416{
1417 {0x00000070, 0x04400000},
1418 {0x00000071, 0x80c01803},
1419 {0x00000072, 0x00004004},
1420 {0x00000073, 0x00000100},
1421 {0x00000074, 0x00ff0000},
1422 {0x00000075, 0x34000000},
1423 {0x00000076, 0x08000014},
1424 {0x00000077, 0x00cc08ec},
1425 {0x00000078, 0x00000400},
1426 {0x00000079, 0x00000000},
1427 {0x0000007a, 0x04090000},
1428 {0x0000007c, 0x00000000},
1429 {0x0000007e, 0x4408a8e8},
1430 {0x0000007f, 0x00000304},
1431 {0x00000080, 0x00000000},
1432 {0x00000082, 0x00000001},
1433 {0x00000083, 0x00000002},
1434 {0x00000084, 0xf3e4f400},
1435 {0x00000085, 0x052024e3},
1436 {0x00000087, 0x00000000},
1437 {0x00000088, 0x01000000},
1438 {0x0000008a, 0x1c0a0000},
1439 {0x0000008b, 0xff010000},
1440 {0x0000008d, 0xffffefff},
1441 {0x0000008e, 0xfff3efff},
1442 {0x0000008f, 0xfff3efbf},
1443 {0x00000092, 0xf7ffffff},
1444 {0x00000093, 0xffffff7f},
1445 {0x00000095, 0x00101101},
1446 {0x00000096, 0x00000fff},
1447 {0x00000097, 0x00116fff},
1448 {0x00000098, 0x60010000},
1449 {0x00000099, 0x10010000},
1450 {0x0000009a, 0x00006000},
1451 {0x0000009b, 0x00001000},
1452 {0x0000009f, 0x00b48000}
1453};
1454
b556b12e
AD		 1455/**
1456 * cik_srbm_select - select specific register instances
1457 *
1458 * @rdev: radeon_device pointer
1459 * @me: selected ME (micro engine)
1460 * @pipe: pipe
1461 * @queue: queue
1462 * @vmid: VMID
1463 *
1464 * Switches the currently active registers instances. Some
1465 * registers are instanced per VMID, others are instanced per
1466 * me/pipe/queue combination.
1467 */
1468static void cik_srbm_select(struct radeon_device *rdev,
1469 u32 me, u32 pipe, u32 queue, u32 vmid)
1470{
1471 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1472 MEID(me & 0x3) |
1473 VMID(vmid & 0xf) |
1474 QUEUEID(queue & 0x7));
1475 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1476}
1477
bc8273fe
AD		 1478/* ucode loading */
1479/**
1480 * ci_mc_load_microcode - load MC ucode into the hw
1481 *
1482 * @rdev: radeon_device pointer
1483 *
1484 * Load the GDDR MC ucode into the hw (CIK).
1485 * Returns 0 on success, error on failure.
1486 */
1487static int ci_mc_load_microcode(struct radeon_device *rdev)
1488{
1489 const __be32 *fw_data;
1490 u32 running, blackout = 0;
1491 u32 *io_mc_regs;
1492 int i, ucode_size, regs_size;
1493
1494 if (!rdev->mc_fw)
1495 return -EINVAL;
1496
1497 switch (rdev->family) {
1498 case CHIP_BONAIRE:
1499 default:
1500 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1501 ucode_size = CIK_MC_UCODE_SIZE;
1502 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1503 break;
1504 }
1505
1506 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1507
1508 if (running == 0) {
1509 if (running) {
1510 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1511 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1512 }
1513
1514 /* reset the engine and set to writable */
1515 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1516 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1517
1518 /* load mc io regs */
1519 for (i = 0; i < regs_size; i++) {
1520 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1521 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1522 }
1523 /* load the MC ucode */
1524 fw_data = (const __be32 *)rdev->mc_fw->data;
1525 for (i = 0; i < ucode_size; i++)
1526 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1527
1528 /* put the engine back into the active state */
1529 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1530 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1531 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1532
1533 /* wait for training to complete */
1534 for (i = 0; i < rdev->usec_timeout; i++) {
1535 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1536 break;
1537 udelay(1);
1538 }
1539 for (i = 0; i < rdev->usec_timeout; i++) {
1540 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1541 break;
1542 udelay(1);
1543 }
1544
1545 if (running)
1546 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1547 }
1548
1549 return 0;
1550}
1551
02c81327
AD		 1552/**
1553 * cik_init_microcode - load ucode images from disk
1554 *
1555 * @rdev: radeon_device pointer
1556 *
1557 * Use the firmware interface to load the ucode images into
1558 * the driver (not loaded into hw).
1559 * Returns 0 on success, error on failure.
1560 */
1561static int cik_init_microcode(struct radeon_device *rdev)
1562{
02c81327
AD		 1563 const char *chip_name;
1564 size_t pfp_req_size, me_req_size, ce_req_size,
21a93e13 1565 mec_req_size, rlc_req_size, mc_req_size,
cc8dbbb4 1566 sdma_req_size, smc_req_size;
02c81327
AD		 1567 char fw_name[30];
1568 int err;
1569
1570 DRM_DEBUG("\n");
1571
02c81327
AD		 1572 switch (rdev->family) {
1573 case CHIP_BONAIRE:
1574 chip_name = "BONAIRE";
1575 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1576 me_req_size = CIK_ME_UCODE_SIZE * 4;
1577 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1578 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1579 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1580 mc_req_size = CIK_MC_UCODE_SIZE * 4;
21a93e13 1581 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 1582 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
02c81327
AD		 1583 break;
1584 case CHIP_KAVERI:
1585 chip_name = "KAVERI";
1586 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1587 me_req_size = CIK_ME_UCODE_SIZE * 4;
1588 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1589 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1590 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 1591 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1592 break;
1593 case CHIP_KABINI:
1594 chip_name = "KABINI";
1595 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1596 me_req_size = CIK_ME_UCODE_SIZE * 4;
1597 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1598 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1599 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 1600 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1601 break;
1602 default: BUG();
1603 }
1604
1605 DRM_INFO("Loading %s Microcode\n", chip_name);
1606
1607 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1608 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
02c81327
AD		 1609 if (err)
1610 goto out;
1611 if (rdev->pfp_fw->size != pfp_req_size) {
1612 printk(KERN_ERR
1613 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1614 rdev->pfp_fw->size, fw_name);
1615 err = -EINVAL;
1616 goto out;
1617 }
1618
1619 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1620 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
02c81327
AD		 1621 if (err)
1622 goto out;
1623 if (rdev->me_fw->size != me_req_size) {
1624 printk(KERN_ERR
1625 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1626 rdev->me_fw->size, fw_name);
1627 err = -EINVAL;
1628 }
1629
1630 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1631 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
02c81327
AD		 1632 if (err)
1633 goto out;
1634 if (rdev->ce_fw->size != ce_req_size) {
1635 printk(KERN_ERR
1636 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1637 rdev->ce_fw->size, fw_name);
1638 err = -EINVAL;
1639 }
1640
1641 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
0a168933 1642 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
02c81327
AD		 1643 if (err)
1644 goto out;
1645 if (rdev->mec_fw->size != mec_req_size) {
1646 printk(KERN_ERR
1647 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1648 rdev->mec_fw->size, fw_name);
1649 err = -EINVAL;
1650 }
1651
1652 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
0a168933 1653 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
02c81327
AD		 1654 if (err)
1655 goto out;
1656 if (rdev->rlc_fw->size != rlc_req_size) {
1657 printk(KERN_ERR
1658 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1659 rdev->rlc_fw->size, fw_name);
1660 err = -EINVAL;
1661 }
1662
21a93e13 1663 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
0a168933 1664 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
21a93e13
AD		 1665 if (err)
1666 goto out;
1667 if (rdev->sdma_fw->size != sdma_req_size) {
1668 printk(KERN_ERR
1669 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1670 rdev->sdma_fw->size, fw_name);
1671 err = -EINVAL;
1672 }
1673
cc8dbbb4 1674 /* No SMC, MC ucode on APUs */
02c81327
AD		 1675 if (!(rdev->flags & RADEON_IS_IGP)) {
1676 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1677 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
02c81327
AD		 1678 if (err)
1679 goto out;
1680 if (rdev->mc_fw->size != mc_req_size) {
1681 printk(KERN_ERR
1682 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1683 rdev->mc_fw->size, fw_name);
1684 err = -EINVAL;
1685 }
cc8dbbb4
AD		 1686
1687 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1688 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1689 if (err) {
1690 printk(KERN_ERR
1691 "smc: error loading firmware \"%s\"\n",
1692 fw_name);
1693 release_firmware(rdev->smc_fw);
1694 rdev->smc_fw = NULL;
1695 } else if (rdev->smc_fw->size != smc_req_size) {
1696 printk(KERN_ERR
1697 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1698 rdev->smc_fw->size, fw_name);
1699 err = -EINVAL;
1700 }
02c81327
AD		 1701 }
1702
1703out:
02c81327
AD		 1704 if (err) {
1705 if (err != -EINVAL)
1706 printk(KERN_ERR
1707 "cik_cp: Failed to load firmware \"%s\"\n",
1708 fw_name);
1709 release_firmware(rdev->pfp_fw);
1710 rdev->pfp_fw = NULL;
1711 release_firmware(rdev->me_fw);
1712 rdev->me_fw = NULL;
1713 release_firmware(rdev->ce_fw);
1714 rdev->ce_fw = NULL;
1715 release_firmware(rdev->rlc_fw);
1716 rdev->rlc_fw = NULL;
1717 release_firmware(rdev->mc_fw);
1718 rdev->mc_fw = NULL;
cc8dbbb4
AD		 1719 release_firmware(rdev->smc_fw);
1720 rdev->smc_fw = NULL;
02c81327
AD		 1721 }
1722 return err;
1723}
1724
8cc1a532
AD		 1725/*
1726 * Core functions
1727 */
1728/**
1729 * cik_tiling_mode_table_init - init the hw tiling table
1730 *
1731 * @rdev: radeon_device pointer
1732 *
1733 * Starting with SI, the tiling setup is done globally in a
1734 * set of 32 tiling modes. Rather than selecting each set of
1735 * parameters per surface as on older asics, we just select
1736 * which index in the tiling table we want to use, and the
1737 * surface uses those parameters (CIK).
1738 */
1739static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1740{
1741 const u32 num_tile_mode_states = 32;
1742 const u32 num_secondary_tile_mode_states = 16;
1743 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1744 u32 num_pipe_configs;
1745 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1746 rdev->config.cik.max_shader_engines;
1747
1748 switch (rdev->config.cik.mem_row_size_in_kb) {
1749 case 1:
1750 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1751 break;
1752 case 2:
1753 default:
1754 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1755 break;
1756 case 4:
1757 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1758 break;
1759 }
1760
1761 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1762 if (num_pipe_configs > 8)
1763 num_pipe_configs = 8; /* ??? */
1764
1765 if (num_pipe_configs == 8) {
1766 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1767 switch (reg_offset) {
1768 case 0:
1769 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1770 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1771 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1773 break;
1774 case 1:
1775 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1776 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1777 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1778 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1779 break;
1780 case 2:
1781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1782 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1783 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1785 break;
1786 case 3:
1787 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1789 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1790 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1791 break;
1792 case 4:
1793 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1794 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1795 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1796 TILE_SPLIT(split_equal_to_row_size));
1797 break;
1798 case 5:
1799 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1800 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1801 break;
1802 case 6:
1803 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1804 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1805 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1806 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1807 break;
1808 case 7:
1809 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1810 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1811 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1812 TILE_SPLIT(split_equal_to_row_size));
1813 break;
1814 case 8:
1815 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1816 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
1817 break;
1818 case 9:
1819 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1820 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1821 break;
1822 case 10:
1823 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1824 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1825 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1826 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1827 break;
1828 case 11:
1829 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1830 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1831 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1832 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1833 break;
1834 case 12:
1835 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1836 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1837 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1838 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1839 break;
1840 case 13:
1841 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1842 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1843 break;
1844 case 14:
1845 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1846 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1847 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1848 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1849 break;
1850 case 16:
1851 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1852 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1853 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1854 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1855 break;
1856 case 17:
1857 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1858 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1859 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1861 break;
1862 case 27:
1863 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1864 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1865 break;
1866 case 28:
1867 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1868 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1869 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1870 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1871 break;
1872 case 29:
1873 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1874 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1875 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1876 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1877 break;
1878 case 30:
1879 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1880 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1881 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1882 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1883 break;
1884 default:
1885 gb_tile_moden = 0;
1886 break;
1887 }
39aee490 1888 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 1889 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1890 }
1891 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1892 switch (reg_offset) {
1893 case 0:
1894 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1895 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1896 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1897 NUM_BANKS(ADDR_SURF_16_BANK));
1898 break;
1899 case 1:
1900 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1901 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1902 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1903 NUM_BANKS(ADDR_SURF_16_BANK));
1904 break;
1905 case 2:
1906 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1907 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1908 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1909 NUM_BANKS(ADDR_SURF_16_BANK));
1910 break;
1911 case 3:
1912 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1913 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1914 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1915 NUM_BANKS(ADDR_SURF_16_BANK));
1916 break;
1917 case 4:
1918 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1919 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1920 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1921 NUM_BANKS(ADDR_SURF_8_BANK));
1922 break;
1923 case 5:
1924 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1925 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1926 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1927 NUM_BANKS(ADDR_SURF_4_BANK));
1928 break;
1929 case 6:
1930 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1931 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1932 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1933 NUM_BANKS(ADDR_SURF_2_BANK));
1934 break;
1935 case 8:
1936 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1937 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1938 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1939 NUM_BANKS(ADDR_SURF_16_BANK));
1940 break;
1941 case 9:
1942 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1943 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1944 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1945 NUM_BANKS(ADDR_SURF_16_BANK));
1946 break;
1947 case 10:
1948 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1949 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1950 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1951 NUM_BANKS(ADDR_SURF_16_BANK));
1952 break;
1953 case 11:
1954 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1957 NUM_BANKS(ADDR_SURF_16_BANK));
1958 break;
1959 case 12:
1960 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1961 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1962 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1963 NUM_BANKS(ADDR_SURF_8_BANK));
1964 break;
1965 case 13:
1966 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1967 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1968 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1969 NUM_BANKS(ADDR_SURF_4_BANK));
1970 break;
1971 case 14:
1972 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1975 NUM_BANKS(ADDR_SURF_2_BANK));
1976 break;
1977 default:
1978 gb_tile_moden = 0;
1979 break;
1980 }
1981 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1982 }
1983 } else if (num_pipe_configs == 4) {
1984 if (num_rbs == 4) {
1985 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1986 switch (reg_offset) {
1987 case 0:
1988 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1989 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1990 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1991 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1992 break;
1993 case 1:
1994 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1995 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1996 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1997 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1998 break;
1999 case 2:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2004 break;
2005 case 3:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2010 break;
2011 case 4:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2015 TILE_SPLIT(split_equal_to_row_size));
2016 break;
2017 case 5:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2020 break;
2021 case 6:
2022 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2023 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2024 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2025 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2026 break;
2027 case 7:
2028 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2029 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2030 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2031 TILE_SPLIT(split_equal_to_row_size));
2032 break;
2033 case 8:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2035 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2036 break;
2037 case 9:
2038 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2039 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2040 break;
2041 case 10:
2042 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2043 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2044 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2045 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2046 break;
2047 case 11:
2048 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2049 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2050 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2051 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2052 break;
2053 case 12:
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2058 break;
2059 case 13:
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2062 break;
2063 case 14:
2064 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2065 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2066 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2067 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2068 break;
2069 case 16:
2070 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2071 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2072 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2073 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2074 break;
2075 case 17:
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2080 break;
2081 case 27:
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2084 break;
2085 case 28:
2086 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2087 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2088 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2089 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2090 break;
2091 case 29:
2092 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2093 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2094 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2095 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2096 break;
2097 case 30:
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102 break;
2103 default:
2104 gb_tile_moden = 0;
2105 break;
2106 }
39aee490 2107 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2108 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2109 }
2110 } else if (num_rbs < 4) {
2111 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2112 switch (reg_offset) {
2113 case 0:
2114 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2115 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2116 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2117 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2118 break;
2119 case 1:
2120 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2121 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2122 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2123 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2124 break;
2125 case 2:
2126 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2127 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2128 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2129 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2130 break;
2131 case 3:
2132 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2133 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2134 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2135 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2136 break;
2137 case 4:
2138 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2139 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2140 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2141 TILE_SPLIT(split_equal_to_row_size));
2142 break;
2143 case 5:
2144 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2145 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2146 break;
2147 case 6:
2148 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2149 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2150 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2151 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2152 break;
2153 case 7:
2154 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2155 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2156 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2157 TILE_SPLIT(split_equal_to_row_size));
2158 break;
2159 case 8:
2160 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2161 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2162 break;
2163 case 9:
2164 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2165 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2166 break;
2167 case 10:
2168 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2169 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2170 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2171 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2172 break;
2173 case 11:
2174 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2175 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2176 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2177 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2178 break;
2179 case 12:
2180 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2181 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2182 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2183 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2184 break;
2185 case 13:
2186 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2187 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2188 break;
2189 case 14:
2190 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2191 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2192 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2193 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2194 break;
2195 case 16:
2196 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2197 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2198 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2199 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2200 break;
2201 case 17:
2202 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2203 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2204 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2205 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2206 break;
2207 case 27:
2208 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2209 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2210 break;
2211 case 28:
2212 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2213 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2214 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2215 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2216 break;
2217 case 29:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2221 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2222 break;
2223 case 30:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2227 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2228 break;
2229 default:
2230 gb_tile_moden = 0;
2231 break;
2232 }
39aee490 2233 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2234 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2235 }
2236 }
2237 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2238 switch (reg_offset) {
2239 case 0:
2240 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2241 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2242 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2243 NUM_BANKS(ADDR_SURF_16_BANK));
2244 break;
2245 case 1:
2246 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2249 NUM_BANKS(ADDR_SURF_16_BANK));
2250 break;
2251 case 2:
2252 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2255 NUM_BANKS(ADDR_SURF_16_BANK));
2256 break;
2257 case 3:
2258 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2259 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2260 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2261 NUM_BANKS(ADDR_SURF_16_BANK));
2262 break;
2263 case 4:
2264 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2267 NUM_BANKS(ADDR_SURF_16_BANK));
2268 break;
2269 case 5:
2270 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2271 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2272 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2273 NUM_BANKS(ADDR_SURF_8_BANK));
2274 break;
2275 case 6:
2276 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2279 NUM_BANKS(ADDR_SURF_4_BANK));
2280 break;
2281 case 8:
2282 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2283 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2284 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2285 NUM_BANKS(ADDR_SURF_16_BANK));
2286 break;
2287 case 9:
2288 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2291 NUM_BANKS(ADDR_SURF_16_BANK));
2292 break;
2293 case 10:
2294 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2297 NUM_BANKS(ADDR_SURF_16_BANK));
2298 break;
2299 case 11:
2300 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2301 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2302 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2303 NUM_BANKS(ADDR_SURF_16_BANK));
2304 break;
2305 case 12:
2306 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2309 NUM_BANKS(ADDR_SURF_16_BANK));
2310 break;
2311 case 13:
2312 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2313 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2314 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2315 NUM_BANKS(ADDR_SURF_8_BANK));
2316 break;
2317 case 14:
2318 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2321 NUM_BANKS(ADDR_SURF_4_BANK));
2322 break;
2323 default:
2324 gb_tile_moden = 0;
2325 break;
2326 }
2327 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2328 }
2329 } else if (num_pipe_configs == 2) {
2330 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2331 switch (reg_offset) {
2332 case 0:
2333 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2334 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2335 PIPE_CONFIG(ADDR_SURF_P2) |
2336 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2337 break;
2338 case 1:
2339 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2340 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2341 PIPE_CONFIG(ADDR_SURF_P2) |
2342 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2343 break;
2344 case 2:
2345 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2346 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2347 PIPE_CONFIG(ADDR_SURF_P2) |
2348 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2349 break;
2350 case 3:
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2352 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2353 PIPE_CONFIG(ADDR_SURF_P2) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2355 break;
2356 case 4:
2357 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2358 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2359 PIPE_CONFIG(ADDR_SURF_P2) |
2360 TILE_SPLIT(split_equal_to_row_size));
2361 break;
2362 case 5:
2363 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2364 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2365 break;
2366 case 6:
2367 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2368 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2369 PIPE_CONFIG(ADDR_SURF_P2) |
2370 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2371 break;
2372 case 7:
2373 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2374 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2375 PIPE_CONFIG(ADDR_SURF_P2) |
2376 TILE_SPLIT(split_equal_to_row_size));
2377 break;
2378 case 8:
2379 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2380 break;
2381 case 9:
2382 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2383 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2384 break;
2385 case 10:
2386 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2387 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2388 PIPE_CONFIG(ADDR_SURF_P2) |
2389 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2390 break;
2391 case 11:
2392 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2393 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2394 PIPE_CONFIG(ADDR_SURF_P2) |
2395 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2396 break;
2397 case 12:
2398 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2399 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2400 PIPE_CONFIG(ADDR_SURF_P2) |
2401 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2402 break;
2403 case 13:
2404 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2405 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2406 break;
2407 case 14:
2408 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2409 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2410 PIPE_CONFIG(ADDR_SURF_P2) |
2411 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2412 break;
2413 case 16:
2414 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2415 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2416 PIPE_CONFIG(ADDR_SURF_P2) |
2417 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2418 break;
2419 case 17:
2420 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2421 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2422 PIPE_CONFIG(ADDR_SURF_P2) |
2423 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2424 break;
2425 case 27:
2426 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2427 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2428 break;
2429 case 28:
2430 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2431 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2432 PIPE_CONFIG(ADDR_SURF_P2) |
2433 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2434 break;
2435 case 29:
2436 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2437 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2438 PIPE_CONFIG(ADDR_SURF_P2) |
2439 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2440 break;
2441 case 30:
2442 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2443 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2444 PIPE_CONFIG(ADDR_SURF_P2) |
2445 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2446 break;
2447 default:
2448 gb_tile_moden = 0;
2449 break;
2450 }
39aee490 2451 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2452 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2453 }
2454 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2455 switch (reg_offset) {
2456 case 0:
2457 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2460 NUM_BANKS(ADDR_SURF_16_BANK));
2461 break;
2462 case 1:
2463 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2466 NUM_BANKS(ADDR_SURF_16_BANK));
2467 break;
2468 case 2:
2469 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2472 NUM_BANKS(ADDR_SURF_16_BANK));
2473 break;
2474 case 3:
2475 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2478 NUM_BANKS(ADDR_SURF_16_BANK));
2479 break;
2480 case 4:
2481 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2484 NUM_BANKS(ADDR_SURF_16_BANK));
2485 break;
2486 case 5:
2487 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2490 NUM_BANKS(ADDR_SURF_16_BANK));
2491 break;
2492 case 6:
2493 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2496 NUM_BANKS(ADDR_SURF_8_BANK));
2497 break;
2498 case 8:
2499 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2500 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2501 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2502 NUM_BANKS(ADDR_SURF_16_BANK));
2503 break;
2504 case 9:
2505 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2508 NUM_BANKS(ADDR_SURF_16_BANK));
2509 break;
2510 case 10:
2511 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2514 NUM_BANKS(ADDR_SURF_16_BANK));
2515 break;
2516 case 11:
2517 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2520 NUM_BANKS(ADDR_SURF_16_BANK));
2521 break;
2522 case 12:
2523 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2524 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2525 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2526 NUM_BANKS(ADDR_SURF_16_BANK));
2527 break;
2528 case 13:
2529 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2530 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2531 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2532 NUM_BANKS(ADDR_SURF_16_BANK));
2533 break;
2534 case 14:
2535 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2536 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2537 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2538 NUM_BANKS(ADDR_SURF_8_BANK));
2539 break;
2540 default:
2541 gb_tile_moden = 0;
2542 break;
2543 }
2544 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2545 }
2546 } else
2547 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2548}
2549
2550/**
2551 * cik_select_se_sh - select which SE, SH to address
2552 *
2553 * @rdev: radeon_device pointer
2554 * @se_num: shader engine to address
2555 * @sh_num: sh block to address
2556 *
2557 * Select which SE, SH combinations to address. Certain
2558 * registers are instanced per SE or SH. 0xffffffff means
2559 * broadcast to all SEs or SHs (CIK).
2560 */
2561static void cik_select_se_sh(struct radeon_device *rdev,
2562 u32 se_num, u32 sh_num)
2563{
2564 u32 data = INSTANCE_BROADCAST_WRITES;
2565
2566 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 2567 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 2568 else if (se_num == 0xffffffff)
2569 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2570 else if (sh_num == 0xffffffff)
2571 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2572 else
2573 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2574 WREG32(GRBM_GFX_INDEX, data);
2575}
2576
2577/**
2578 * cik_create_bitmask - create a bitmask
2579 *
2580 * @bit_width: length of the mask
2581 *
2582 * create a variable length bit mask (CIK).
2583 * Returns the bitmask.
2584 */
2585static u32 cik_create_bitmask(u32 bit_width)
2586{
2587 u32 i, mask = 0;
2588
2589 for (i = 0; i < bit_width; i++) {
2590 mask <<= 1;
2591 mask |= 1;
2592 }
2593 return mask;
2594}
2595
2596/**
2597 * cik_select_se_sh - select which SE, SH to address
2598 *
2599 * @rdev: radeon_device pointer
2600 * @max_rb_num: max RBs (render backends) for the asic
2601 * @se_num: number of SEs (shader engines) for the asic
2602 * @sh_per_se: number of SH blocks per SE for the asic
2603 *
2604 * Calculates the bitmask of disabled RBs (CIK).
2605 * Returns the disabled RB bitmask.
2606 */
2607static u32 cik_get_rb_disabled(struct radeon_device *rdev,
2608 u32 max_rb_num, u32 se_num,
2609 u32 sh_per_se)
2610{
2611 u32 data, mask;
2612
2613 data = RREG32(CC_RB_BACKEND_DISABLE);
2614 if (data & 1)
2615 data &= BACKEND_DISABLE_MASK;
2616 else
2617 data = 0;
2618 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2619
2620 data >>= BACKEND_DISABLE_SHIFT;
2621
2622 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
2623
2624 return data & mask;
2625}
2626
2627/**
2628 * cik_setup_rb - setup the RBs on the asic
2629 *
2630 * @rdev: radeon_device pointer
2631 * @se_num: number of SEs (shader engines) for the asic
2632 * @sh_per_se: number of SH blocks per SE for the asic
2633 * @max_rb_num: max RBs (render backends) for the asic
2634 *
2635 * Configures per-SE/SH RB registers (CIK).
2636 */
2637static void cik_setup_rb(struct radeon_device *rdev,
2638 u32 se_num, u32 sh_per_se,
2639 u32 max_rb_num)
2640{
2641 int i, j;
2642 u32 data, mask;
2643 u32 disabled_rbs = 0;
2644 u32 enabled_rbs = 0;
2645
2646 for (i = 0; i < se_num; i++) {
2647 for (j = 0; j < sh_per_se; j++) {
2648 cik_select_se_sh(rdev, i, j);
2649 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2650 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
2651 }
2652 }
2653 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2654
2655 mask = 1;
2656 for (i = 0; i < max_rb_num; i++) {
2657 if (!(disabled_rbs & mask))
2658 enabled_rbs |= mask;
2659 mask <<= 1;
2660 }
2661
2662 for (i = 0; i < se_num; i++) {
2663 cik_select_se_sh(rdev, i, 0xffffffff);
2664 data = 0;
2665 for (j = 0; j < sh_per_se; j++) {
2666 switch (enabled_rbs & 3) {
2667 case 1:
2668 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2669 break;
2670 case 2:
2671 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2672 break;
2673 case 3:
2674 default:
2675 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2676 break;
2677 }
2678 enabled_rbs >>= 2;
2679 }
2680 WREG32(PA_SC_RASTER_CONFIG, data);
2681 }
2682 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2683}
2684
2685/**
2686 * cik_gpu_init - setup the 3D engine
2687 *
2688 * @rdev: radeon_device pointer
2689 *
2690 * Configures the 3D engine and tiling configuration
2691 * registers so that the 3D engine is usable.
2692 */
2693static void cik_gpu_init(struct radeon_device *rdev)
2694{
2695 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
2696 u32 mc_shared_chmap, mc_arb_ramcfg;
2697 u32 hdp_host_path_cntl;
2698 u32 tmp;
2699 int i, j;
2700
2701 switch (rdev->family) {
2702 case CHIP_BONAIRE:
2703 rdev->config.cik.max_shader_engines = 2;
2704 rdev->config.cik.max_tile_pipes = 4;
2705 rdev->config.cik.max_cu_per_sh = 7;
2706 rdev->config.cik.max_sh_per_se = 1;
2707 rdev->config.cik.max_backends_per_se = 2;
2708 rdev->config.cik.max_texture_channel_caches = 4;
2709 rdev->config.cik.max_gprs = 256;
2710 rdev->config.cik.max_gs_threads = 32;
2711 rdev->config.cik.max_hw_contexts = 8;
2712
2713 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2714 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2715 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2716 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2717 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2718 break;
2719 case CHIP_KAVERI:
b2e4c70a
AD		 2720 rdev->config.cik.max_shader_engines = 1;
2721 rdev->config.cik.max_tile_pipes = 4;
2722 if ((rdev->pdev->device == 0x1304) ||
2723 (rdev->pdev->device == 0x1305) ||
2724 (rdev->pdev->device == 0x130C) ||
2725 (rdev->pdev->device == 0x130F) ||
2726 (rdev->pdev->device == 0x1310) ||
2727 (rdev->pdev->device == 0x1311) ||
2728 (rdev->pdev->device == 0x131C)) {
2729 rdev->config.cik.max_cu_per_sh = 8;
2730 rdev->config.cik.max_backends_per_se = 2;
2731 } else if ((rdev->pdev->device == 0x1309) ||
2732 (rdev->pdev->device == 0x130A) ||
2733 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 2734 (rdev->pdev->device == 0x1313) ||
2735 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 2736 rdev->config.cik.max_cu_per_sh = 6;
2737 rdev->config.cik.max_backends_per_se = 2;
2738 } else if ((rdev->pdev->device == 0x1306) ||
2739 (rdev->pdev->device == 0x1307) ||
2740 (rdev->pdev->device == 0x130B) ||
2741 (rdev->pdev->device == 0x130E) ||
2742 (rdev->pdev->device == 0x1315) ||
2743 (rdev->pdev->device == 0x131B)) {
2744 rdev->config.cik.max_cu_per_sh = 4;
2745 rdev->config.cik.max_backends_per_se = 1;
2746 } else {
2747 rdev->config.cik.max_cu_per_sh = 3;
2748 rdev->config.cik.max_backends_per_se = 1;
2749 }
2750 rdev->config.cik.max_sh_per_se = 1;
2751 rdev->config.cik.max_texture_channel_caches = 4;
2752 rdev->config.cik.max_gprs = 256;
2753 rdev->config.cik.max_gs_threads = 16;
2754 rdev->config.cik.max_hw_contexts = 8;
2755
2756 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2757 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2758 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2759 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2760 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 2761 break;
2762 case CHIP_KABINI:
2763 default:
2764 rdev->config.cik.max_shader_engines = 1;
2765 rdev->config.cik.max_tile_pipes = 2;
2766 rdev->config.cik.max_cu_per_sh = 2;
2767 rdev->config.cik.max_sh_per_se = 1;
2768 rdev->config.cik.max_backends_per_se = 1;
2769 rdev->config.cik.max_texture_channel_caches = 2;
2770 rdev->config.cik.max_gprs = 256;
2771 rdev->config.cik.max_gs_threads = 16;
2772 rdev->config.cik.max_hw_contexts = 8;
2773
2774 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2775 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2776 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2777 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2778 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2779 break;
2780 }
2781
2782 /* Initialize HDP */
2783 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2784 WREG32((0x2c14 + j), 0x00000000);
2785 WREG32((0x2c18 + j), 0x00000000);
2786 WREG32((0x2c1c + j), 0x00000000);
2787 WREG32((0x2c20 + j), 0x00000000);
2788 WREG32((0x2c24 + j), 0x00000000);
2789 }
2790
2791 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2792
2793 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2794
2795 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2796 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2797
2798 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
2799 rdev->config.cik.mem_max_burst_length_bytes = 256;
2800 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2801 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2802 if (rdev->config.cik.mem_row_size_in_kb > 4)
2803 rdev->config.cik.mem_row_size_in_kb = 4;
2804 /* XXX use MC settings? */
2805 rdev->config.cik.shader_engine_tile_size = 32;
2806 rdev->config.cik.num_gpus = 1;
2807 rdev->config.cik.multi_gpu_tile_size = 64;
2808
2809 /* fix up row size */
2810 gb_addr_config &= ~ROW_SIZE_MASK;
2811 switch (rdev->config.cik.mem_row_size_in_kb) {
2812 case 1:
2813 default:
2814 gb_addr_config |= ROW_SIZE(0);
2815 break;
2816 case 2:
2817 gb_addr_config |= ROW_SIZE(1);
2818 break;
2819 case 4:
2820 gb_addr_config |= ROW_SIZE(2);
2821 break;
2822 }
2823
2824 /* setup tiling info dword. gb_addr_config is not adequate since it does
2825 * not have bank info, so create a custom tiling dword.
2826 * bits 3:0 num_pipes
2827 * bits 7:4 num_banks
2828 * bits 11:8 group_size
2829 * bits 15:12 row_size
2830 */
2831 rdev->config.cik.tile_config = 0;
2832 switch (rdev->config.cik.num_tile_pipes) {
2833 case 1:
2834 rdev->config.cik.tile_config |= (0 << 0);
2835 break;
2836 case 2:
2837 rdev->config.cik.tile_config |= (1 << 0);
2838 break;
2839 case 4:
2840 rdev->config.cik.tile_config |= (2 << 0);
2841 break;
2842 case 8:
2843 default:
2844 /* XXX what about 12? */
2845 rdev->config.cik.tile_config |= (3 << 0);
2846 break;
2847 }
a537314e
MD		 2848 rdev->config.cik.tile_config |=
2849 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 2850 rdev->config.cik.tile_config |=
2851 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2852 rdev->config.cik.tile_config |=
2853 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2854
2855 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2856 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2857 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 2858 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
2859 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 2860 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2861 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2862 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 2863
2864 cik_tiling_mode_table_init(rdev);
2865
2866 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
2867 rdev->config.cik.max_sh_per_se,
2868 rdev->config.cik.max_backends_per_se);
2869
2870 /* set HW defaults for 3D engine */
2871 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2872
2873 WREG32(SX_DEBUG_1, 0x20);
2874
2875 WREG32(TA_CNTL_AUX, 0x00010000);
2876
2877 tmp = RREG32(SPI_CONFIG_CNTL);
2878 tmp |= 0x03000000;
2879 WREG32(SPI_CONFIG_CNTL, tmp);
2880
2881 WREG32(SQ_CONFIG, 1);
2882
2883 WREG32(DB_DEBUG, 0);
2884
2885 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
2886 tmp |= 0x00000400;
2887 WREG32(DB_DEBUG2, tmp);
2888
2889 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
2890 tmp |= 0x00020200;
2891 WREG32(DB_DEBUG3, tmp);
2892
2893 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
2894 tmp |= 0x00018208;
2895 WREG32(CB_HW_CONTROL, tmp);
2896
2897 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2898
2899 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
2900 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
2901 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
2902 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
2903
2904 WREG32(VGT_NUM_INSTANCES, 1);
2905
2906 WREG32(CP_PERFMON_CNTL, 0);
2907
2908 WREG32(SQ_CONFIG, 0);
2909
2910 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2911 FORCE_EOV_MAX_REZ_CNT(255)));
2912
2913 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2914 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2915
2916 WREG32(VGT_GS_VERTEX_REUSE, 16);
2917 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2918
2919 tmp = RREG32(HDP_MISC_CNTL);
2920 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2921 WREG32(HDP_MISC_CNTL, tmp);
2922
2923 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2924 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2925
2926 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2927 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
2928
2929 udelay(50);
2930}
2931
2cae3bc3
AD		 2932/*
2933 * GPU scratch registers helpers function.
2934 */
2935/**
2936 * cik_scratch_init - setup driver info for CP scratch regs
2937 *
2938 * @rdev: radeon_device pointer
2939 *
2940 * Set up the number and offset of the CP scratch registers.
2941 * NOTE: use of CP scratch registers is a legacy inferface and
2942 * is not used by default on newer asics (r6xx+). On newer asics,
2943 * memory buffers are used for fences rather than scratch regs.
2944 */
2945static void cik_scratch_init(struct radeon_device *rdev)
2946{
2947 int i;
2948
2949 rdev->scratch.num_reg = 7;
2950 rdev->scratch.reg_base = SCRATCH_REG0;
2951 for (i = 0; i < rdev->scratch.num_reg; i++) {
2952 rdev->scratch.free[i] = true;
2953 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2954 }
2955}
2956
fbc832c7
AD		 2957/**
2958 * cik_ring_test - basic gfx ring test
2959 *
2960 * @rdev: radeon_device pointer
2961 * @ring: radeon_ring structure holding ring information
2962 *
2963 * Allocate a scratch register and write to it using the gfx ring (CIK).
2964 * Provides a basic gfx ring test to verify that the ring is working.
2965 * Used by cik_cp_gfx_resume();
2966 * Returns 0 on success, error on failure.
2967 */
2968int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2969{
2970 uint32_t scratch;
2971 uint32_t tmp = 0;
2972 unsigned i;
2973 int r;
2974
2975 r = radeon_scratch_get(rdev, &scratch);
2976 if (r) {
2977 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2978 return r;
2979 }
2980 WREG32(scratch, 0xCAFEDEAD);
2981 r = radeon_ring_lock(rdev, ring, 3);
2982 if (r) {
2983 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2984 radeon_scratch_free(rdev, scratch);
2985 return r;
2986 }
2987 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2988 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
2989 radeon_ring_write(ring, 0xDEADBEEF);
2990 radeon_ring_unlock_commit(rdev, ring);
963e81f9 2991
fbc832c7
AD		 2992 for (i = 0; i < rdev->usec_timeout; i++) {
2993 tmp = RREG32(scratch);
2994 if (tmp == 0xDEADBEEF)
2995 break;
2996 DRM_UDELAY(1);
2997 }
2998 if (i < rdev->usec_timeout) {
2999 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3000 } else {
3001 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3002 ring->idx, scratch, tmp);
3003 r = -EINVAL;
3004 }
3005 radeon_scratch_free(rdev, scratch);
3006 return r;
3007}
3008
2cae3bc3 3009/**
b07fdd38 3010 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3011 *
3012 * @rdev: radeon_device pointer
3013 * @fence: radeon fence object
3014 *
3015 * Emits a fence sequnce number on the gfx ring and flushes
3016 * GPU caches.
3017 */
b07fdd38
AD		 3018void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3019 struct radeon_fence *fence)
2cae3bc3
AD		 3020{
3021 struct radeon_ring *ring = &rdev->ring[fence->ring];
3022 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3023
3024 /* EVENT_WRITE_EOP - flush caches, send int */
3025 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3026 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3027 EOP_TC_ACTION_EN |
3028 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3029 EVENT_INDEX(5)));
3030 radeon_ring_write(ring, addr & 0xfffffffc);
3031 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3032 radeon_ring_write(ring, fence->seq);
3033 radeon_ring_write(ring, 0);
3034 /* HDP flush */
3035 /* We should be using the new WAIT_REG_MEM special op packet here
3036 * but it causes the CP to hang
3037 */
3038 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3039 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3040 WRITE_DATA_DST_SEL(0)));
3041 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3042 radeon_ring_write(ring, 0);
3043 radeon_ring_write(ring, 0);
3044}
3045
b07fdd38
AD		 3046/**
3047 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3048 *
3049 * @rdev: radeon_device pointer
3050 * @fence: radeon fence object
3051 *
3052 * Emits a fence sequnce number on the compute ring and flushes
3053 * GPU caches.
3054 */
3055void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3056 struct radeon_fence *fence)
3057{
3058 struct radeon_ring *ring = &rdev->ring[fence->ring];
3059 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3060
3061 /* RELEASE_MEM - flush caches, send int */
3062 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3063 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3064 EOP_TC_ACTION_EN |
3065 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3066 EVENT_INDEX(5)));
3067 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3068 radeon_ring_write(ring, addr & 0xfffffffc);
3069 radeon_ring_write(ring, upper_32_bits(addr));
3070 radeon_ring_write(ring, fence->seq);
3071 radeon_ring_write(ring, 0);
3072 /* HDP flush */
3073 /* We should be using the new WAIT_REG_MEM special op packet here
3074 * but it causes the CP to hang
3075 */
3076 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3077 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3078 WRITE_DATA_DST_SEL(0)));
3079 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3080 radeon_ring_write(ring, 0);
3081 radeon_ring_write(ring, 0);
3082}
3083
2cae3bc3
AD		 3084void cik_semaphore_ring_emit(struct radeon_device *rdev,
3085 struct radeon_ring *ring,
3086 struct radeon_semaphore *semaphore,
3087 bool emit_wait)
3088{
3089 uint64_t addr = semaphore->gpu_addr;
3090 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3091
3092 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3093 radeon_ring_write(ring, addr & 0xffffffff);
3094 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3095}
3096
c9dbd705
AD		 3097/**
3098 * cik_copy_cpdma - copy pages using the CP DMA engine
3099 *
3100 * @rdev: radeon_device pointer
3101 * @src_offset: src GPU address
3102 * @dst_offset: dst GPU address
3103 * @num_gpu_pages: number of GPU pages to xfer
3104 * @fence: radeon fence object
3105 *
3106 * Copy GPU paging using the CP DMA engine (CIK+).
3107 * Used by the radeon ttm implementation to move pages if
3108 * registered as the asic copy callback.
3109 */
3110int cik_copy_cpdma(struct radeon_device *rdev,
3111 uint64_t src_offset, uint64_t dst_offset,
3112 unsigned num_gpu_pages,
3113 struct radeon_fence **fence)
3114{
3115 struct radeon_semaphore *sem = NULL;
3116 int ring_index = rdev->asic->copy.blit_ring_index;
3117 struct radeon_ring *ring = &rdev->ring[ring_index];
3118 u32 size_in_bytes, cur_size_in_bytes, control;
3119 int i, num_loops;
3120 int r = 0;
3121
3122 r = radeon_semaphore_create(rdev, &sem);
3123 if (r) {
3124 DRM_ERROR("radeon: moving bo (%d).\n", r);
3125 return r;
3126 }
3127
3128 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3129 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3130 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3131 if (r) {
3132 DRM_ERROR("radeon: moving bo (%d).\n", r);
3133 radeon_semaphore_free(rdev, &sem, NULL);
3134 return r;
3135 }
3136
3137 if (radeon_fence_need_sync(*fence, ring->idx)) {
3138 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3139 ring->idx);
3140 radeon_fence_note_sync(*fence, ring->idx);
3141 } else {
3142 radeon_semaphore_free(rdev, &sem, NULL);
3143 }
3144
3145 for (i = 0; i < num_loops; i++) {
3146 cur_size_in_bytes = size_in_bytes;
3147 if (cur_size_in_bytes > 0x1fffff)
3148 cur_size_in_bytes = 0x1fffff;
3149 size_in_bytes -= cur_size_in_bytes;
3150 control = 0;
3151 if (size_in_bytes == 0)
3152 control |= PACKET3_DMA_DATA_CP_SYNC;
3153 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3154 radeon_ring_write(ring, control);
3155 radeon_ring_write(ring, lower_32_bits(src_offset));
3156 radeon_ring_write(ring, upper_32_bits(src_offset));
3157 radeon_ring_write(ring, lower_32_bits(dst_offset));
3158 radeon_ring_write(ring, upper_32_bits(dst_offset));
3159 radeon_ring_write(ring, cur_size_in_bytes);
3160 src_offset += cur_size_in_bytes;
3161 dst_offset += cur_size_in_bytes;
3162 }
3163
3164 r = radeon_fence_emit(rdev, fence, ring->idx);
3165 if (r) {
3166 radeon_ring_unlock_undo(rdev, ring);
3167 return r;
3168 }
3169
3170 radeon_ring_unlock_commit(rdev, ring);
3171 radeon_semaphore_free(rdev, &sem, *fence);
3172
3173 return r;
3174}
3175
2cae3bc3
AD		 3176/*
3177 * IB stuff
3178 */
3179/**
3180 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3181 *
3182 * @rdev: radeon_device pointer
3183 * @ib: radeon indirect buffer object
3184 *
3185 * Emits an DE (drawing engine) or CE (constant engine) IB
3186 * on the gfx ring. IBs are usually generated by userspace
3187 * acceleration drivers and submitted to the kernel for
3188 * sheduling on the ring. This function schedules the IB
3189 * on the gfx ring for execution by the GPU.
3190 */
3191void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3192{
3193 struct radeon_ring *ring = &rdev->ring[ib->ring];
3194 u32 header, control = INDIRECT_BUFFER_VALID;
3195
3196 if (ib->is_const_ib) {
3197 /* set switch buffer packet before const IB */
3198 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3199 radeon_ring_write(ring, 0);
3200
3201 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3202 } else {
3203 u32 next_rptr;
3204 if (ring->rptr_save_reg) {
3205 next_rptr = ring->wptr + 3 + 4;
3206 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3207 radeon_ring_write(ring, ((ring->rptr_save_reg -
3208 PACKET3_SET_UCONFIG_REG_START) >> 2));
3209 radeon_ring_write(ring, next_rptr);
3210 } else if (rdev->wb.enabled) {
3211 next_rptr = ring->wptr + 5 + 4;
3212 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3213 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3214 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3215 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3216 radeon_ring_write(ring, next_rptr);
3217 }
3218
3219 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3220 }
3221
3222 control |= ib->length_dw |
3223 (ib->vm ? (ib->vm->id << 24) : 0);
3224
3225 radeon_ring_write(ring, header);
3226 radeon_ring_write(ring,
3227#ifdef __BIG_ENDIAN
3228 (2 << 0) |
3229#endif
3230 (ib->gpu_addr & 0xFFFFFFFC));
3231 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3232 radeon_ring_write(ring, control);
3233}
3234
fbc832c7
AD		 3235/**
3236 * cik_ib_test - basic gfx ring IB test
3237 *
3238 * @rdev: radeon_device pointer
3239 * @ring: radeon_ring structure holding ring information
3240 *
3241 * Allocate an IB and execute it on the gfx ring (CIK).
3242 * Provides a basic gfx ring test to verify that IBs are working.
3243 * Returns 0 on success, error on failure.
3244 */
3245int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3246{
3247 struct radeon_ib ib;
3248 uint32_t scratch;
3249 uint32_t tmp = 0;
3250 unsigned i;
3251 int r;
3252
3253 r = radeon_scratch_get(rdev, &scratch);
3254 if (r) {
3255 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3256 return r;
3257 }
3258 WREG32(scratch, 0xCAFEDEAD);
3259 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3260 if (r) {
3261 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3262 return r;
3263 }
3264 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3265 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3266 ib.ptr[2] = 0xDEADBEEF;
3267 ib.length_dw = 3;
3268 r = radeon_ib_schedule(rdev, &ib, NULL);
3269 if (r) {
3270 radeon_scratch_free(rdev, scratch);
3271 radeon_ib_free(rdev, &ib);
3272 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3273 return r;
3274 }
3275 r = radeon_fence_wait(ib.fence, false);
3276 if (r) {
3277 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3278 return r;
3279 }
3280 for (i = 0; i < rdev->usec_timeout; i++) {
3281 tmp = RREG32(scratch);
3282 if (tmp == 0xDEADBEEF)
3283 break;
3284 DRM_UDELAY(1);
3285 }
3286 if (i < rdev->usec_timeout) {
3287 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3288 } else {
3289 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3290 scratch, tmp);
3291 r = -EINVAL;
3292 }
3293 radeon_scratch_free(rdev, scratch);
3294 radeon_ib_free(rdev, &ib);
3295 return r;
3296}
3297
841cf442
AD		 3298/*
3299 * CP.
3300 * On CIK, gfx and compute now have independant command processors.
3301 *
3302 * GFX
3303 * Gfx consists of a single ring and can process both gfx jobs and
3304 * compute jobs. The gfx CP consists of three microengines (ME):
3305 * PFP - Pre-Fetch Parser
3306 * ME - Micro Engine
3307 * CE - Constant Engine
3308 * The PFP and ME make up what is considered the Drawing Engine (DE).
3309 * The CE is an asynchronous engine used for updating buffer desciptors
3310 * used by the DE so that they can be loaded into cache in parallel
3311 * while the DE is processing state update packets.
3312 *
3313 * Compute
3314 * The compute CP consists of two microengines (ME):
3315 * MEC1 - Compute MicroEngine 1
3316 * MEC2 - Compute MicroEngine 2
3317 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3318 * The queues are exposed to userspace and are programmed directly
3319 * by the compute runtime.
3320 */
3321/**
3322 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3323 *
3324 * @rdev: radeon_device pointer
3325 * @enable: enable or disable the MEs
3326 *
3327 * Halts or unhalts the gfx MEs.
3328 */
3329static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3330{
3331 if (enable)
3332 WREG32(CP_ME_CNTL, 0);
3333 else {
3334 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3335 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3336 }
3337 udelay(50);
3338}
3339
3340/**
3341 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3342 *
3343 * @rdev: radeon_device pointer
3344 *
3345 * Loads the gfx PFP, ME, and CE ucode.
3346 * Returns 0 for success, -EINVAL if the ucode is not available.
3347 */
3348static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3349{
3350 const __be32 *fw_data;
3351 int i;
3352
3353 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3354 return -EINVAL;
3355
3356 cik_cp_gfx_enable(rdev, false);
3357
3358 /* PFP */
3359 fw_data = (const __be32 *)rdev->pfp_fw->data;
3360 WREG32(CP_PFP_UCODE_ADDR, 0);
3361 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3362 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3363 WREG32(CP_PFP_UCODE_ADDR, 0);
3364
3365 /* CE */
3366 fw_data = (const __be32 *)rdev->ce_fw->data;
3367 WREG32(CP_CE_UCODE_ADDR, 0);
3368 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3369 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3370 WREG32(CP_CE_UCODE_ADDR, 0);
3371
3372 /* ME */
3373 fw_data = (const __be32 *)rdev->me_fw->data;
3374 WREG32(CP_ME_RAM_WADDR, 0);
3375 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3376 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3377 WREG32(CP_ME_RAM_WADDR, 0);
3378
3379 WREG32(CP_PFP_UCODE_ADDR, 0);
3380 WREG32(CP_CE_UCODE_ADDR, 0);
3381 WREG32(CP_ME_RAM_WADDR, 0);
3382 WREG32(CP_ME_RAM_RADDR, 0);
3383 return 0;
3384}
3385
3386/**
3387 * cik_cp_gfx_start - start the gfx ring
3388 *
3389 * @rdev: radeon_device pointer
3390 *
3391 * Enables the ring and loads the clear state context and other
3392 * packets required to init the ring.
3393 * Returns 0 for success, error for failure.
3394 */
3395static int cik_cp_gfx_start(struct radeon_device *rdev)
3396{
3397 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3398 int r, i;
3399
3400 /* init the CP */
3401 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3402 WREG32(CP_ENDIAN_SWAP, 0);
3403 WREG32(CP_DEVICE_ID, 1);
3404
3405 cik_cp_gfx_enable(rdev, true);
3406
3407 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3408 if (r) {
3409 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3410 return r;
3411 }
3412
3413 /* init the CE partitions. CE only used for gfx on CIK */
3414 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3415 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3416 radeon_ring_write(ring, 0xc000);
3417 radeon_ring_write(ring, 0xc000);
3418
3419 /* setup clear context state */
3420 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3421 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3422
3423 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3424 radeon_ring_write(ring, 0x80000000);
3425 radeon_ring_write(ring, 0x80000000);
3426
3427 for (i = 0; i < cik_default_size; i++)
3428 radeon_ring_write(ring, cik_default_state[i]);
3429
3430 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3431 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3432
3433 /* set clear context state */
3434 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3435 radeon_ring_write(ring, 0);
3436
3437 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3438 radeon_ring_write(ring, 0x00000316);
3439 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3440 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3441
3442 radeon_ring_unlock_commit(rdev, ring);
3443
3444 return 0;
3445}
3446
3447/**
3448 * cik_cp_gfx_fini - stop the gfx ring
3449 *
3450 * @rdev: radeon_device pointer
3451 *
3452 * Stop the gfx ring and tear down the driver ring
3453 * info.
3454 */
3455static void cik_cp_gfx_fini(struct radeon_device *rdev)
3456{
3457 cik_cp_gfx_enable(rdev, false);
3458 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3459}
3460
3461/**
3462 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3463 *
3464 * @rdev: radeon_device pointer
3465 *
3466 * Program the location and size of the gfx ring buffer
3467 * and test it to make sure it's working.
3468 * Returns 0 for success, error for failure.
3469 */
3470static int cik_cp_gfx_resume(struct radeon_device *rdev)
3471{
3472 struct radeon_ring *ring;
3473 u32 tmp;
3474 u32 rb_bufsz;
3475 u64 rb_addr;
3476 int r;
3477
3478 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3479 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3480
3481 /* Set the write pointer delay */
3482 WREG32(CP_RB_WPTR_DELAY, 0);
3483
3484 /* set the RB to use vmid 0 */
3485 WREG32(CP_RB_VMID, 0);
3486
3487 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3488
3489 /* ring 0 - compute and gfx */
3490 /* Set ring buffer size */
3491 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3492 rb_bufsz = order_base_2(ring->ring_size / 8);
3493 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 3494#ifdef __BIG_ENDIAN
3495 tmp |= BUF_SWAP_32BIT;
3496#endif
3497 WREG32(CP_RB0_CNTL, tmp);
3498
3499 /* Initialize the ring buffer's read and write pointers */
3500 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3501 ring->wptr = 0;
3502 WREG32(CP_RB0_WPTR, ring->wptr);
3503
3504 /* set the wb address wether it's enabled or not */
3505 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3506 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3507
3508 /* scratch register shadowing is no longer supported */
3509 WREG32(SCRATCH_UMSK, 0);
3510
3511 if (!rdev->wb.enabled)
3512 tmp |= RB_NO_UPDATE;
3513
3514 mdelay(1);
3515 WREG32(CP_RB0_CNTL, tmp);
3516
3517 rb_addr = ring->gpu_addr >> 8;
3518 WREG32(CP_RB0_BASE, rb_addr);
3519 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
3520
3521 ring->rptr = RREG32(CP_RB0_RPTR);
3522
3523 /* start the ring */
3524 cik_cp_gfx_start(rdev);
3525 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3526 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3527 if (r) {
3528 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3529 return r;
3530 }
3531 return 0;
3532}
3533
963e81f9
AD		 3534u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
3535 struct radeon_ring *ring)
3536{
3537 u32 rptr;
3538
3539
3540
3541 if (rdev->wb.enabled) {
3542 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
3543 } else {
f61d5b46 3544 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3545 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3546 rptr = RREG32(CP_HQD_PQ_RPTR);
3547 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 3548 mutex_unlock(&rdev->srbm_mutex);
963e81f9 3549 }
963e81f9
AD		 3550
3551 return rptr;
3552}
3553
3554u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
3555 struct radeon_ring *ring)
3556{
3557 u32 wptr;
3558
3559 if (rdev->wb.enabled) {
3560 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
3561 } else {
f61d5b46 3562 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3563 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3564 wptr = RREG32(CP_HQD_PQ_WPTR);
3565 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 3566 mutex_unlock(&rdev->srbm_mutex);
963e81f9 3567 }
963e81f9
AD		 3568
3569 return wptr;
3570}
3571
3572void cik_compute_ring_set_wptr(struct radeon_device *rdev,
3573 struct radeon_ring *ring)
3574{
2e1e6dad
CK		 3575 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
3576 WDOORBELL32(ring->doorbell_offset, ring->wptr);
963e81f9
AD		 3577}
3578
841cf442
AD		 3579/**
3580 * cik_cp_compute_enable - enable/disable the compute CP MEs
3581 *
3582 * @rdev: radeon_device pointer
3583 * @enable: enable or disable the MEs
3584 *
3585 * Halts or unhalts the compute MEs.
3586 */
3587static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
3588{
3589 if (enable)
3590 WREG32(CP_MEC_CNTL, 0);
3591 else
3592 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
3593 udelay(50);
3594}
3595
3596/**
3597 * cik_cp_compute_load_microcode - load the compute CP ME ucode
3598 *
3599 * @rdev: radeon_device pointer
3600 *
3601 * Loads the compute MEC1&2 ucode.
3602 * Returns 0 for success, -EINVAL if the ucode is not available.
3603 */
3604static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
3605{
3606 const __be32 *fw_data;
3607 int i;
3608
3609 if (!rdev->mec_fw)
3610 return -EINVAL;
3611
3612 cik_cp_compute_enable(rdev, false);
3613
3614 /* MEC1 */
3615 fw_data = (const __be32 *)rdev->mec_fw->data;
3616 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3617 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3618 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
3619 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3620
3621 if (rdev->family == CHIP_KAVERI) {
3622 /* MEC2 */
3623 fw_data = (const __be32 *)rdev->mec_fw->data;
3624 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3625 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3626 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
3627 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3628 }
3629
3630 return 0;
3631}
3632
3633/**
3634 * cik_cp_compute_start - start the compute queues
3635 *
3636 * @rdev: radeon_device pointer
3637 *
3638 * Enable the compute queues.
3639 * Returns 0 for success, error for failure.
3640 */
3641static int cik_cp_compute_start(struct radeon_device *rdev)
3642{
963e81f9
AD		 3643 cik_cp_compute_enable(rdev, true);
3644
841cf442
AD		 3645 return 0;
3646}
3647
3648/**
3649 * cik_cp_compute_fini - stop the compute queues
3650 *
3651 * @rdev: radeon_device pointer
3652 *
3653 * Stop the compute queues and tear down the driver queue
3654 * info.
3655 */
3656static void cik_cp_compute_fini(struct radeon_device *rdev)
3657{
963e81f9
AD		 3658 int i, idx, r;
3659
841cf442 3660 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 3661
3662 for (i = 0; i < 2; i++) {
3663 if (i == 0)
3664 idx = CAYMAN_RING_TYPE_CP1_INDEX;
3665 else
3666 idx = CAYMAN_RING_TYPE_CP2_INDEX;
3667
3668 if (rdev->ring[idx].mqd_obj) {
3669 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3670 if (unlikely(r != 0))
3671 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
3672
3673 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
3674 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3675
3676 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
3677 rdev->ring[idx].mqd_obj = NULL;
3678 }
3679 }
841cf442
AD		 3680}
3681
963e81f9
AD		 3682static void cik_mec_fini(struct radeon_device *rdev)
3683{
3684 int r;
3685
3686 if (rdev->mec.hpd_eop_obj) {
3687 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3688 if (unlikely(r != 0))
3689 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
3690 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
3691 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3692
3693 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
3694 rdev->mec.hpd_eop_obj = NULL;
3695 }
3696}
3697
3698#define MEC_HPD_SIZE 2048
3699
3700static int cik_mec_init(struct radeon_device *rdev)
3701{
3702 int r;
3703 u32 *hpd;
3704
3705 /*
3706 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
3707 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
3708 */
3709 if (rdev->family == CHIP_KAVERI)
3710 rdev->mec.num_mec = 2;
3711 else
3712 rdev->mec.num_mec = 1;
3713 rdev->mec.num_pipe = 4;
3714 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
3715
3716 if (rdev->mec.hpd_eop_obj == NULL) {
3717 r = radeon_bo_create(rdev,
3718 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
3719 PAGE_SIZE, true,
3720 RADEON_GEM_DOMAIN_GTT, NULL,
3721 &rdev->mec.hpd_eop_obj);
3722 if (r) {
3723 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
3724 return r;
3725 }
3726 }
3727
3728 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3729 if (unlikely(r != 0)) {
3730 cik_mec_fini(rdev);
3731 return r;
3732 }
3733 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
3734 &rdev->mec.hpd_eop_gpu_addr);
3735 if (r) {
3736 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
3737 cik_mec_fini(rdev);
3738 return r;
3739 }
3740 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
3741 if (r) {
3742 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
3743 cik_mec_fini(rdev);
3744 return r;
3745 }
3746
3747 /* clear memory. Not sure if this is required or not */
3748 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
3749
3750 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
3751 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3752
3753 return 0;
3754}
3755
3756struct hqd_registers
3757{
3758 u32 cp_mqd_base_addr;
3759 u32 cp_mqd_base_addr_hi;
3760 u32 cp_hqd_active;
3761 u32 cp_hqd_vmid;
3762 u32 cp_hqd_persistent_state;
3763 u32 cp_hqd_pipe_priority;
3764 u32 cp_hqd_queue_priority;
3765 u32 cp_hqd_quantum;
3766 u32 cp_hqd_pq_base;
3767 u32 cp_hqd_pq_base_hi;
3768 u32 cp_hqd_pq_rptr;
3769 u32 cp_hqd_pq_rptr_report_addr;
3770 u32 cp_hqd_pq_rptr_report_addr_hi;
3771 u32 cp_hqd_pq_wptr_poll_addr;
3772 u32 cp_hqd_pq_wptr_poll_addr_hi;
3773 u32 cp_hqd_pq_doorbell_control;
3774 u32 cp_hqd_pq_wptr;
3775 u32 cp_hqd_pq_control;
3776 u32 cp_hqd_ib_base_addr;
3777 u32 cp_hqd_ib_base_addr_hi;
3778 u32 cp_hqd_ib_rptr;
3779 u32 cp_hqd_ib_control;
3780 u32 cp_hqd_iq_timer;
3781 u32 cp_hqd_iq_rptr;
3782 u32 cp_hqd_dequeue_request;
3783 u32 cp_hqd_dma_offload;
3784 u32 cp_hqd_sema_cmd;
3785 u32 cp_hqd_msg_type;
3786 u32 cp_hqd_atomic0_preop_lo;
3787 u32 cp_hqd_atomic0_preop_hi;
3788 u32 cp_hqd_atomic1_preop_lo;
3789 u32 cp_hqd_atomic1_preop_hi;
3790 u32 cp_hqd_hq_scheduler0;
3791 u32 cp_hqd_hq_scheduler1;
3792 u32 cp_mqd_control;
3793};
3794
3795struct bonaire_mqd
3796{
3797 u32 header;
3798 u32 dispatch_initiator;
3799 u32 dimensions[3];
3800 u32 start_idx[3];
3801 u32 num_threads[3];
3802 u32 pipeline_stat_enable;
3803 u32 perf_counter_enable;
3804 u32 pgm[2];
3805 u32 tba[2];
3806 u32 tma[2];
3807 u32 pgm_rsrc[2];
3808 u32 vmid;
3809 u32 resource_limits;
3810 u32 static_thread_mgmt01[2];
3811 u32 tmp_ring_size;
3812 u32 static_thread_mgmt23[2];
3813 u32 restart[3];
3814 u32 thread_trace_enable;
3815 u32 reserved1;
3816 u32 user_data[16];
3817 u32 vgtcs_invoke_count[2];
3818 struct hqd_registers queue_state;
3819 u32 dequeue_cntr;
3820 u32 interrupt_queue[64];
3821};
3822
841cf442
AD		 3823/**
3824 * cik_cp_compute_resume - setup the compute queue registers
3825 *
3826 * @rdev: radeon_device pointer
3827 *
3828 * Program the compute queues and test them to make sure they
3829 * are working.
3830 * Returns 0 for success, error for failure.
3831 */
3832static int cik_cp_compute_resume(struct radeon_device *rdev)
3833{
963e81f9
AD		 3834 int r, i, idx;
3835 u32 tmp;
3836 bool use_doorbell = true;
3837 u64 hqd_gpu_addr;
3838 u64 mqd_gpu_addr;
3839 u64 eop_gpu_addr;
3840 u64 wb_gpu_addr;
3841 u32 *buf;
3842 struct bonaire_mqd *mqd;
841cf442 3843
841cf442
AD		 3844 r = cik_cp_compute_start(rdev);
3845 if (r)
3846 return r;
963e81f9
AD		 3847
3848 /* fix up chicken bits */
3849 tmp = RREG32(CP_CPF_DEBUG);
3850 tmp |= (1 << 23);
3851 WREG32(CP_CPF_DEBUG, tmp);
3852
3853 /* init the pipes */
f61d5b46 3854 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3855 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
3856 int me = (i < 4) ? 1 : 2;
3857 int pipe = (i < 4) ? i : (i - 4);
3858
3859 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
3860
3861 cik_srbm_select(rdev, me, pipe, 0, 0);
3862
3863 /* write the EOP addr */
3864 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
3865 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
3866
3867 /* set the VMID assigned */
3868 WREG32(CP_HPD_EOP_VMID, 0);
3869
3870 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3871 tmp = RREG32(CP_HPD_EOP_CONTROL);
3872 tmp &= ~EOP_SIZE_MASK;
b72a8925 3873 tmp |= order_base_2(MEC_HPD_SIZE / 8);
963e81f9
AD		 3874 WREG32(CP_HPD_EOP_CONTROL, tmp);
3875 }
3876 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 3877 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 3878
3879 /* init the queues. Just two for now. */
3880 for (i = 0; i < 2; i++) {
3881 if (i == 0)
3882 idx = CAYMAN_RING_TYPE_CP1_INDEX;
3883 else
3884 idx = CAYMAN_RING_TYPE_CP2_INDEX;
3885
3886 if (rdev->ring[idx].mqd_obj == NULL) {
3887 r = radeon_bo_create(rdev,
3888 sizeof(struct bonaire_mqd),
3889 PAGE_SIZE, true,
3890 RADEON_GEM_DOMAIN_GTT, NULL,
3891 &rdev->ring[idx].mqd_obj);
3892 if (r) {
3893 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
3894 return r;
3895 }
3896 }
3897
3898 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3899 if (unlikely(r != 0)) {
3900 cik_cp_compute_fini(rdev);
3901 return r;
3902 }
3903 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
3904 &mqd_gpu_addr);
3905 if (r) {
3906 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
3907 cik_cp_compute_fini(rdev);
3908 return r;
3909 }
3910 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
3911 if (r) {
3912 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
3913 cik_cp_compute_fini(rdev);
3914 return r;
3915 }
3916
3917 /* doorbell offset */
3918 rdev->ring[idx].doorbell_offset =
3919 (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
3920
3921 /* init the mqd struct */
3922 memset(buf, 0, sizeof(struct bonaire_mqd));
3923
3924 mqd = (struct bonaire_mqd *)buf;
3925 mqd->header = 0xC0310800;
3926 mqd->static_thread_mgmt01[0] = 0xffffffff;
3927 mqd->static_thread_mgmt01[1] = 0xffffffff;
3928 mqd->static_thread_mgmt23[0] = 0xffffffff;
3929 mqd->static_thread_mgmt23[1] = 0xffffffff;
3930
f61d5b46 3931 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3932 cik_srbm_select(rdev, rdev->ring[idx].me,
3933 rdev->ring[idx].pipe,
3934 rdev->ring[idx].queue, 0);
3935
3936 /* disable wptr polling */
3937 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
3938 tmp &= ~WPTR_POLL_EN;
3939 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
3940
3941 /* enable doorbell? */
3942 mqd->queue_state.cp_hqd_pq_doorbell_control =
3943 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3944 if (use_doorbell)
3945 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3946 else
3947 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
3948 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3949 mqd->queue_state.cp_hqd_pq_doorbell_control);
3950
3951 /* disable the queue if it's active */
3952 mqd->queue_state.cp_hqd_dequeue_request = 0;
3953 mqd->queue_state.cp_hqd_pq_rptr = 0;
3954 mqd->queue_state.cp_hqd_pq_wptr= 0;
3955 if (RREG32(CP_HQD_ACTIVE) & 1) {
3956 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
3957 for (i = 0; i < rdev->usec_timeout; i++) {
3958 if (!(RREG32(CP_HQD_ACTIVE) & 1))
3959 break;
3960 udelay(1);
3961 }
3962 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
3963 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
3964 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3965 }
3966
3967 /* set the pointer to the MQD */
3968 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
3969 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
3970 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
3971 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
3972 /* set MQD vmid to 0 */
3973 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
3974 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
3975 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
3976
3977 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3978 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
3979 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
3980 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3981 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
3982 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
3983
3984 /* set up the HQD, this is similar to CP_RB0_CNTL */
3985 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
3986 mqd->queue_state.cp_hqd_pq_control &=
3987 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
3988
3989 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 3990 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 3991 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 3992 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 3993#ifdef __BIG_ENDIAN
3994 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
3995#endif
3996 mqd->queue_state.cp_hqd_pq_control &=
3997 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
3998 mqd->queue_state.cp_hqd_pq_control |=
3999 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4000 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4001
4002 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4003 if (i == 0)
4004 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4005 else
4006 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4007 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4008 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4009 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4010 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4011 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4012
4013 /* set the wb address wether it's enabled or not */
4014 if (i == 0)
4015 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4016 else
4017 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4018 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4019 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4020 upper_32_bits(wb_gpu_addr) & 0xffff;
4021 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4022 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4023 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4024 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4025
4026 /* enable the doorbell if requested */
4027 if (use_doorbell) {
4028 mqd->queue_state.cp_hqd_pq_doorbell_control =
4029 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4030 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4031 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4032 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
4033 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4034 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4035 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4036
4037 } else {
4038 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4039 }
4040 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4041 mqd->queue_state.cp_hqd_pq_doorbell_control);
4042
4043 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4044 rdev->ring[idx].wptr = 0;
4045 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4046 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4047 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4048 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4049
4050 /* set the vmid for the queue */
4051 mqd->queue_state.cp_hqd_vmid = 0;
4052 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4053
4054 /* activate the queue */
4055 mqd->queue_state.cp_hqd_active = 1;
4056 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4057
4058 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4059 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4060
4061 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4062 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4063
4064 rdev->ring[idx].ready = true;
4065 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4066 if (r)
4067 rdev->ring[idx].ready = false;
4068 }
4069
841cf442
AD		 4070 return 0;
4071}
4072
841cf442
AD		 4073static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4074{
4075 cik_cp_gfx_enable(rdev, enable);
4076 cik_cp_compute_enable(rdev, enable);
4077}
4078
841cf442
AD		 4079static int cik_cp_load_microcode(struct radeon_device *rdev)
4080{
4081 int r;
4082
4083 r = cik_cp_gfx_load_microcode(rdev);
4084 if (r)
4085 return r;
4086 r = cik_cp_compute_load_microcode(rdev);
4087 if (r)
4088 return r;
4089
4090 return 0;
4091}
4092
841cf442
AD		 4093static void cik_cp_fini(struct radeon_device *rdev)
4094{
4095 cik_cp_gfx_fini(rdev);
4096 cik_cp_compute_fini(rdev);
4097}
4098
841cf442
AD		 4099static int cik_cp_resume(struct radeon_device *rdev)
4100{
4101 int r;
4102
4214faf6
AD		 4103 cik_enable_gui_idle_interrupt(rdev, false);
4104
841cf442
AD		 4105 r = cik_cp_load_microcode(rdev);
4106 if (r)
4107 return r;
4108
4109 r = cik_cp_gfx_resume(rdev);
4110 if (r)
4111 return r;
4112 r = cik_cp_compute_resume(rdev);
4113 if (r)
4114 return r;
4115
4214faf6
AD		 4116 cik_enable_gui_idle_interrupt(rdev, true);
4117
841cf442
AD		 4118 return 0;
4119}
4120
cc066715 4121static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 4122{
6f2043ce
AD		 4123 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4124 RREG32(GRBM_STATUS));
4125 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4126 RREG32(GRBM_STATUS2));
4127 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4128 RREG32(GRBM_STATUS_SE0));
4129 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4130 RREG32(GRBM_STATUS_SE1));
4131 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4132 RREG32(GRBM_STATUS_SE2));
4133 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4134 RREG32(GRBM_STATUS_SE3));
4135 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4136 RREG32(SRBM_STATUS));
4137 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4138 RREG32(SRBM_STATUS2));
cc066715
AD		 4139 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4140 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4141 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4142 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 4143 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4144 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4145 RREG32(CP_STALLED_STAT1));
4146 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4147 RREG32(CP_STALLED_STAT2));
4148 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4149 RREG32(CP_STALLED_STAT3));
4150 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4151 RREG32(CP_CPF_BUSY_STAT));
4152 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4153 RREG32(CP_CPF_STALLED_STAT1));
4154 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4155 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4156 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4157 RREG32(CP_CPC_STALLED_STAT1));
4158 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 4159}
6f2043ce 4160
21a93e13 4161/**
cc066715 4162 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 4163 *
4164 * @rdev: radeon_device pointer
21a93e13 4165 *
cc066715
AD		 4166 * Check which blocks are busy and return the relevant reset
4167 * mask to be used by cik_gpu_soft_reset().
4168 * Returns a mask of the blocks to be reset.
21a93e13 4169 */
2483b4ea 4170u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 4171{
cc066715
AD		 4172 u32 reset_mask = 0;
4173 u32 tmp;
21a93e13 4174
cc066715
AD		 4175 /* GRBM_STATUS */
4176 tmp = RREG32(GRBM_STATUS);
4177 if (tmp & (PA_BUSY | SC_BUSY |
4178 BCI_BUSY | SX_BUSY |
4179 TA_BUSY | VGT_BUSY |
4180 DB_BUSY | CB_BUSY |
4181 GDS_BUSY | SPI_BUSY |
4182 IA_BUSY | IA_BUSY_NO_DMA))
4183 reset_mask |= RADEON_RESET_GFX;
21a93e13 4184
cc066715
AD		 4185 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4186 reset_mask |= RADEON_RESET_CP;
21a93e13 4187
cc066715
AD		 4188 /* GRBM_STATUS2 */
4189 tmp = RREG32(GRBM_STATUS2);
4190 if (tmp & RLC_BUSY)
4191 reset_mask |= RADEON_RESET_RLC;
21a93e13 4192
cc066715
AD		 4193 /* SDMA0_STATUS_REG */
4194 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4195 if (!(tmp & SDMA_IDLE))
4196 reset_mask |= RADEON_RESET_DMA;
21a93e13 4197
cc066715
AD		 4198 /* SDMA1_STATUS_REG */
4199 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4200 if (!(tmp & SDMA_IDLE))
4201 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4202
cc066715
AD		 4203 /* SRBM_STATUS2 */
4204 tmp = RREG32(SRBM_STATUS2);
4205 if (tmp & SDMA_BUSY)
4206 reset_mask |= RADEON_RESET_DMA;
21a93e13 4207
cc066715
AD		 4208 if (tmp & SDMA1_BUSY)
4209 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4210
cc066715
AD		 4211 /* SRBM_STATUS */
4212 tmp = RREG32(SRBM_STATUS);
21a93e13 4213
cc066715
AD		 4214 if (tmp & IH_BUSY)
4215 reset_mask |= RADEON_RESET_IH;
21a93e13 4216
cc066715
AD		 4217 if (tmp & SEM_BUSY)
4218 reset_mask |= RADEON_RESET_SEM;
21a93e13 4219
cc066715
AD		 4220 if (tmp & GRBM_RQ_PENDING)
4221 reset_mask |= RADEON_RESET_GRBM;
21a93e13 4222
cc066715
AD		 4223 if (tmp & VMC_BUSY)
4224 reset_mask |= RADEON_RESET_VMC;
21a93e13 4225
cc066715
AD		 4226 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4227 MCC_BUSY | MCD_BUSY))
4228 reset_mask |= RADEON_RESET_MC;
21a93e13 4229
cc066715
AD		 4230 if (evergreen_is_display_hung(rdev))
4231 reset_mask |= RADEON_RESET_DISPLAY;
4232
4233 /* Skip MC reset as it's mostly likely not hung, just busy */
4234 if (reset_mask & RADEON_RESET_MC) {
4235 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4236 reset_mask &= ~RADEON_RESET_MC;
21a93e13 4237 }
cc066715
AD		 4238
4239 return reset_mask;
21a93e13
AD		 4240}
4241
4242/**
cc066715 4243 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 4244 *
4245 * @rdev: radeon_device pointer
cc066715 4246 * @reset_mask: mask of which blocks to reset
21a93e13 4247 *
cc066715 4248 * Soft reset the blocks specified in @reset_mask.
21a93e13 4249 */
cc066715 4250static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 4251{
6f2043ce 4252 struct evergreen_mc_save save;
cc066715
AD		 4253 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4254 u32 tmp;
21a93e13 4255
cc066715
AD		 4256 if (reset_mask == 0)
4257 return;
21a93e13 4258
cc066715 4259 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 4260
cc066715
AD		 4261 cik_print_gpu_status_regs(rdev);
4262 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4263 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4264 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4265 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 4266
cc066715
AD		 4267 /* stop the rlc */
4268 cik_rlc_stop(rdev);
21a93e13 4269
cc066715
AD		 4270 /* Disable GFX parsing/prefetching */
4271 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 4272
cc066715
AD		 4273 /* Disable MEC parsing/prefetching */
4274 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 4275
cc066715
AD		 4276 if (reset_mask & RADEON_RESET_DMA) {
4277 /* sdma0 */
4278 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4279 tmp |= SDMA_HALT;
4280 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4281 }
4282 if (reset_mask & RADEON_RESET_DMA1) {
4283 /* sdma1 */
4284 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4285 tmp |= SDMA_HALT;
4286 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4287 }
21a93e13 4288
6f2043ce 4289 evergreen_mc_stop(rdev, &save);
cc066715 4290 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 4291 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4292 }
21a93e13 4293
cc066715
AD		 4294 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4295 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 4296
cc066715
AD		 4297 if (reset_mask & RADEON_RESET_CP) {
4298 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 4299
cc066715
AD		 4300 srbm_soft_reset |= SOFT_RESET_GRBM;
4301 }
21a93e13 4302
cc066715
AD		 4303 if (reset_mask & RADEON_RESET_DMA)
4304 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 4305
cc066715
AD		 4306 if (reset_mask & RADEON_RESET_DMA1)
4307 srbm_soft_reset |= SOFT_RESET_SDMA1;
4308
4309 if (reset_mask & RADEON_RESET_DISPLAY)
4310 srbm_soft_reset |= SOFT_RESET_DC;
4311
4312 if (reset_mask & RADEON_RESET_RLC)
4313 grbm_soft_reset |= SOFT_RESET_RLC;
4314
4315 if (reset_mask & RADEON_RESET_SEM)
4316 srbm_soft_reset |= SOFT_RESET_SEM;
4317
4318 if (reset_mask & RADEON_RESET_IH)
4319 srbm_soft_reset |= SOFT_RESET_IH;
4320
4321 if (reset_mask & RADEON_RESET_GRBM)
4322 srbm_soft_reset |= SOFT_RESET_GRBM;
4323
4324 if (reset_mask & RADEON_RESET_VMC)
4325 srbm_soft_reset |= SOFT_RESET_VMC;
4326
4327 if (!(rdev->flags & RADEON_IS_IGP)) {
4328 if (reset_mask & RADEON_RESET_MC)
4329 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 4330 }
4331
cc066715
AD		 4332 if (grbm_soft_reset) {
4333 tmp = RREG32(GRBM_SOFT_RESET);
4334 tmp |= grbm_soft_reset;
4335 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4336 WREG32(GRBM_SOFT_RESET, tmp);
4337 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 4338
cc066715 4339 udelay(50);
21a93e13 4340
cc066715
AD		 4341 tmp &= ~grbm_soft_reset;
4342 WREG32(GRBM_SOFT_RESET, tmp);
4343 tmp = RREG32(GRBM_SOFT_RESET);
4344 }
21a93e13 4345
cc066715
AD		 4346 if (srbm_soft_reset) {
4347 tmp = RREG32(SRBM_SOFT_RESET);
4348 tmp |= srbm_soft_reset;
4349 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4350 WREG32(SRBM_SOFT_RESET, tmp);
4351 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 4352
cc066715 4353 udelay(50);
21a93e13 4354
cc066715
AD		 4355 tmp &= ~srbm_soft_reset;
4356 WREG32(SRBM_SOFT_RESET, tmp);
4357 tmp = RREG32(SRBM_SOFT_RESET);
4358 }
21a93e13 4359
6f2043ce
AD		 4360 /* Wait a little for things to settle down */
4361 udelay(50);
21a93e13 4362
6f2043ce 4363 evergreen_mc_resume(rdev, &save);
cc066715
AD		 4364 udelay(50);
4365
4366 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 4367}
4368
4369/**
cc066715 4370 * cik_asic_reset - soft reset GPU
21a93e13
AD		 4371 *
4372 * @rdev: radeon_device pointer
4373 *
cc066715
AD		 4374 * Look up which blocks are hung and attempt
4375 * to reset them.
6f2043ce 4376 * Returns 0 for success.
21a93e13 4377 */
6f2043ce 4378int cik_asic_reset(struct radeon_device *rdev)
21a93e13 4379{
cc066715 4380 u32 reset_mask;
21a93e13 4381
cc066715 4382 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 4383
cc066715
AD		 4384 if (reset_mask)
4385 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 4386
cc066715 4387 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 4388
cc066715
AD		 4389 reset_mask = cik_gpu_check_soft_reset(rdev);
4390
4391 if (!reset_mask)
4392 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 4393
4394 return 0;
4395}
4396
4397/**
cc066715 4398 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 4399 *
4400 * @rdev: radeon_device pointer
cc066715 4401 * @ring: radeon_ring structure holding ring information
21a93e13 4402 *
cc066715
AD		 4403 * Check if the 3D engine is locked up (CIK).
4404 * Returns true if the engine is locked, false if not.
21a93e13 4405 */
cc066715 4406bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 4407{
cc066715 4408 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 4409
cc066715
AD		 4410 if (!(reset_mask & (RADEON_RESET_GFX |
4411 RADEON_RESET_COMPUTE |
4412 RADEON_RESET_CP))) {
4413 radeon_ring_lockup_update(ring);
4414 return false;
21a93e13 4415 }
cc066715
AD		 4416 /* force CP activities */
4417 radeon_ring_force_activity(rdev, ring);
4418 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 4419}
4420
1c49165d 4421/* MC */
21a93e13 4422/**
1c49165d 4423 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 4424 *
4425 * @rdev: radeon_device pointer
21a93e13 4426 *
1c49165d
AD		 4427 * Set the location of vram, gart, and AGP in the GPU's
4428 * physical address space (CIK).
21a93e13 4429 */
1c49165d 4430static void cik_mc_program(struct radeon_device *rdev)
21a93e13 4431{
1c49165d 4432 struct evergreen_mc_save save;
21a93e13 4433 u32 tmp;
1c49165d 4434 int i, j;
21a93e13 4435
1c49165d
AD		 4436 /* Initialize HDP */
4437 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4438 WREG32((0x2c14 + j), 0x00000000);
4439 WREG32((0x2c18 + j), 0x00000000);
4440 WREG32((0x2c1c + j), 0x00000000);
4441 WREG32((0x2c20 + j), 0x00000000);
4442 WREG32((0x2c24 + j), 0x00000000);
21a93e13 4443 }
1c49165d 4444 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 4445
1c49165d
AD		 4446 evergreen_mc_stop(rdev, &save);
4447 if (radeon_mc_wait_for_idle(rdev)) {
4448 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 4449 }
1c49165d
AD		 4450 /* Lockout access through VGA aperture*/
4451 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4452 /* Update configuration */
4453 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4454 rdev->mc.vram_start >> 12);
4455 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4456 rdev->mc.vram_end >> 12);
4457 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4458 rdev->vram_scratch.gpu_addr >> 12);
4459 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4460 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4461 WREG32(MC_VM_FB_LOCATION, tmp);
4462 /* XXX double check these! */
4463 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4464 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4465 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4466 WREG32(MC_VM_AGP_BASE, 0);
4467 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4468 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4469 if (radeon_mc_wait_for_idle(rdev)) {
4470 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 4471 }
1c49165d
AD		 4472 evergreen_mc_resume(rdev, &save);
4473 /* we need to own VRAM, so turn off the VGA renderer here
4474 * to stop it overwriting our objects */
4475 rv515_vga_render_disable(rdev);
21a93e13
AD		 4476}
4477
4478/**
1c49165d 4479 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 4480 *
4481 * @rdev: radeon_device pointer
21a93e13 4482 *
1c49165d
AD		 4483 * Look up the amount of vram, vram width, and decide how to place
4484 * vram and gart within the GPU's physical address space (CIK).
4485 * Returns 0 for success.
21a93e13 4486 */
1c49165d 4487static int cik_mc_init(struct radeon_device *rdev)
21a93e13 4488{
1c49165d
AD		 4489 u32 tmp;
4490 int chansize, numchan;
21a93e13 4491
1c49165d
AD		 4492 /* Get VRAM informations */
4493 rdev->mc.vram_is_ddr = true;
4494 tmp = RREG32(MC_ARB_RAMCFG);
4495 if (tmp & CHANSIZE_MASK) {
4496 chansize = 64;
21a93e13 4497 } else {
1c49165d 4498 chansize = 32;
21a93e13 4499 }
1c49165d
AD		 4500 tmp = RREG32(MC_SHARED_CHMAP);
4501 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4502 case 0:
4503 default:
4504 numchan = 1;
4505 break;
4506 case 1:
4507 numchan = 2;
4508 break;
4509 case 2:
4510 numchan = 4;
4511 break;
4512 case 3:
4513 numchan = 8;
4514 break;
4515 case 4:
4516 numchan = 3;
4517 break;
4518 case 5:
4519 numchan = 6;
4520 break;
4521 case 6:
4522 numchan = 10;
4523 break;
4524 case 7:
4525 numchan = 12;
4526 break;
4527 case 8:
4528 numchan = 16;
4529 break;
4530 }
4531 rdev->mc.vram_width = numchan * chansize;
4532 /* Could aper size report 0 ? */
4533 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4534 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4535 /* size in MB on si */
13c5bfda
AD		 4536 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
4537 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 4538 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4539 si_vram_gtt_location(rdev, &rdev->mc);
4540 radeon_update_bandwidth_info(rdev);
4541
4542 return 0;
4543}
4544
4545/*
4546 * GART
4547 * VMID 0 is the physical GPU addresses as used by the kernel.
4548 * VMIDs 1-15 are used for userspace clients and are handled
4549 * by the radeon vm/hsa code.
4550 */
4551/**
4552 * cik_pcie_gart_tlb_flush - gart tlb flush callback
4553 *
4554 * @rdev: radeon_device pointer
4555 *
4556 * Flush the TLB for the VMID 0 page table (CIK).
4557 */
4558void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
4559{
4560 /* flush hdp cache */
4561 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
4562
4563 /* bits 0-15 are the VM contexts0-15 */
4564 WREG32(VM_INVALIDATE_REQUEST, 0x1);
4565}
4566
4567/**
4568 * cik_pcie_gart_enable - gart enable
4569 *
4570 * @rdev: radeon_device pointer
4571 *
4572 * This sets up the TLBs, programs the page tables for VMID0,
4573 * sets up the hw for VMIDs 1-15 which are allocated on
4574 * demand, and sets up the global locations for the LDS, GDS,
4575 * and GPUVM for FSA64 clients (CIK).
4576 * Returns 0 for success, errors for failure.
4577 */
4578static int cik_pcie_gart_enable(struct radeon_device *rdev)
4579{
4580 int r, i;
4581
4582 if (rdev->gart.robj == NULL) {
4583 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4584 return -EINVAL;
4585 }
4586 r = radeon_gart_table_vram_pin(rdev);
4587 if (r)
4588 return r;
4589 radeon_gart_restore(rdev);
4590 /* Setup TLB control */
4591 WREG32(MC_VM_MX_L1_TLB_CNTL,
4592 (0xA << 7) |
4593 ENABLE_L1_TLB |
4594 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4595 ENABLE_ADVANCED_DRIVER_MODEL |
4596 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4597 /* Setup L2 cache */
4598 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4599 ENABLE_L2_FRAGMENT_PROCESSING |
4600 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4601 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4602 EFFECTIVE_L2_QUEUE_SIZE(7) |
4603 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4604 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4605 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4606 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4607 /* setup context0 */
4608 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4609 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4610 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4611 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4612 (u32)(rdev->dummy_page.addr >> 12));
4613 WREG32(VM_CONTEXT0_CNTL2, 0);
4614 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4615 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4616
4617 WREG32(0x15D4, 0);
4618 WREG32(0x15D8, 0);
4619 WREG32(0x15DC, 0);
4620
4621 /* empty context1-15 */
4622 /* FIXME start with 4G, once using 2 level pt switch to full
4623 * vm size space
4624 */
4625 /* set vm size, must be a multiple of 4 */
4626 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4627 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4628 for (i = 1; i < 16; i++) {
4629 if (i < 8)
4630 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4631 rdev->gart.table_addr >> 12);
4632 else
4633 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4634 rdev->gart.table_addr >> 12);
4635 }
4636
4637 /* enable context1-15 */
4638 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4639 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 4640 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 4641 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
a00024b0
AD		 4642 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4643 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4644 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4645 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4646 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4647 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4648 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4649 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4650 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4651 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4652 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4653 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d
AD		 4654
4655 /* TC cache setup ??? */
4656 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
4657 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
4658 WREG32(TC_CFG_L1_STORE_POLICY, 0);
4659
4660 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
4661 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
4662 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
4663 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
4664 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
4665
4666 WREG32(TC_CFG_L1_VOLATILE, 0);
4667 WREG32(TC_CFG_L2_VOLATILE, 0);
4668
4669 if (rdev->family == CHIP_KAVERI) {
4670 u32 tmp = RREG32(CHUB_CONTROL);
4671 tmp &= ~BYPASS_VM;
4672 WREG32(CHUB_CONTROL, tmp);
4673 }
4674
4675 /* XXX SH_MEM regs */
4676 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 4677 mutex_lock(&rdev->srbm_mutex);
1c49165d 4678 for (i = 0; i < 16; i++) {
b556b12e 4679 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 4680 /* CP and shaders */
1c49165d
AD		 4681 WREG32(SH_MEM_CONFIG, 0);
4682 WREG32(SH_MEM_APE1_BASE, 1);
4683 WREG32(SH_MEM_APE1_LIMIT, 0);
4684 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 4685 /* SDMA GFX */
4686 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
4687 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
4688 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
4689 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
4690 /* XXX SDMA RLC - todo */
1c49165d 4691 }
b556b12e 4692 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4693 mutex_unlock(&rdev->srbm_mutex);
1c49165d
AD		 4694
4695 cik_pcie_gart_tlb_flush(rdev);
4696 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4697 (unsigned)(rdev->mc.gtt_size >> 20),
4698 (unsigned long long)rdev->gart.table_addr);
4699 rdev->gart.ready = true;
4700 return 0;
4701}
4702
4703/**
4704 * cik_pcie_gart_disable - gart disable
4705 *
4706 * @rdev: radeon_device pointer
4707 *
4708 * This disables all VM page table (CIK).
4709 */
4710static void cik_pcie_gart_disable(struct radeon_device *rdev)
4711{
4712 /* Disable all tables */
4713 WREG32(VM_CONTEXT0_CNTL, 0);
4714 WREG32(VM_CONTEXT1_CNTL, 0);
4715 /* Setup TLB control */
4716 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4717 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4718 /* Setup L2 cache */
4719 WREG32(VM_L2_CNTL,
4720 ENABLE_L2_FRAGMENT_PROCESSING |
4721 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4722 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4723 EFFECTIVE_L2_QUEUE_SIZE(7) |
4724 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4725 WREG32(VM_L2_CNTL2, 0);
4726 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4727 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4728 radeon_gart_table_vram_unpin(rdev);
4729}
4730
4731/**
4732 * cik_pcie_gart_fini - vm fini callback
4733 *
4734 * @rdev: radeon_device pointer
4735 *
4736 * Tears down the driver GART/VM setup (CIK).
4737 */
4738static void cik_pcie_gart_fini(struct radeon_device *rdev)
4739{
4740 cik_pcie_gart_disable(rdev);
4741 radeon_gart_table_vram_free(rdev);
4742 radeon_gart_fini(rdev);
4743}
4744
4745/* vm parser */
4746/**
4747 * cik_ib_parse - vm ib_parse callback
4748 *
4749 * @rdev: radeon_device pointer
4750 * @ib: indirect buffer pointer
4751 *
4752 * CIK uses hw IB checking so this is a nop (CIK).
4753 */
4754int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4755{
4756 return 0;
4757}
4758
4759/*
4760 * vm
4761 * VMID 0 is the physical GPU addresses as used by the kernel.
4762 * VMIDs 1-15 are used for userspace clients and are handled
4763 * by the radeon vm/hsa code.
4764 */
4765/**
4766 * cik_vm_init - cik vm init callback
4767 *
4768 * @rdev: radeon_device pointer
4769 *
4770 * Inits cik specific vm parameters (number of VMs, base of vram for
4771 * VMIDs 1-15) (CIK).
4772 * Returns 0 for success.
4773 */
4774int cik_vm_init(struct radeon_device *rdev)
4775{
4776 /* number of VMs */
4777 rdev->vm_manager.nvm = 16;
4778 /* base offset of vram pages */
4779 if (rdev->flags & RADEON_IS_IGP) {
4780 u64 tmp = RREG32(MC_VM_FB_OFFSET);
4781 tmp <<= 22;
4782 rdev->vm_manager.vram_base_offset = tmp;
4783 } else
4784 rdev->vm_manager.vram_base_offset = 0;
4785
4786 return 0;
4787}
4788
4789/**
4790 * cik_vm_fini - cik vm fini callback
4791 *
4792 * @rdev: radeon_device pointer
4793 *
4794 * Tear down any asic specific VM setup (CIK).
4795 */
4796void cik_vm_fini(struct radeon_device *rdev)
4797{
4798}
4799
3ec7d11b
AD		 4800/**
4801 * cik_vm_decode_fault - print human readable fault info
4802 *
4803 * @rdev: radeon_device pointer
4804 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4805 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4806 *
4807 * Print human readable fault information (CIK).
4808 */
4809static void cik_vm_decode_fault(struct radeon_device *rdev,
4810 u32 status, u32 addr, u32 mc_client)
4811{
4812 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4813 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4814 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 4815 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
4816 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 4817
328a50c7 4818 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 4819 protections, vmid, addr,
4820 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 4821 block, mc_client, mc_id);
3ec7d11b
AD		 4822}
4823
f96ab484
AD		 4824/**
4825 * cik_vm_flush - cik vm flush using the CP
4826 *
4827 * @rdev: radeon_device pointer
4828 *
4829 * Update the page table base and flush the VM TLB
4830 * using the CP (CIK).
4831 */
4832void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4833{
4834 struct radeon_ring *ring = &rdev->ring[ridx];
4835
4836 if (vm == NULL)
4837 return;
4838
4839 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4840 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4841 WRITE_DATA_DST_SEL(0)));
4842 if (vm->id < 8) {
4843 radeon_ring_write(ring,
4844 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4845 } else {
4846 radeon_ring_write(ring,
4847 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4848 }
4849 radeon_ring_write(ring, 0);
4850 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4851
4852 /* update SH_MEM_* regs */
4853 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4854 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4855 WRITE_DATA_DST_SEL(0)));
4856 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4857 radeon_ring_write(ring, 0);
4858 radeon_ring_write(ring, VMID(vm->id));
4859
4860 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4861 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4862 WRITE_DATA_DST_SEL(0)));
4863 radeon_ring_write(ring, SH_MEM_BASES >> 2);
4864 radeon_ring_write(ring, 0);
4865
4866 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
4867 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
4868 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
4869 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 4870
f96ab484
AD		 4871 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4872 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4873 WRITE_DATA_DST_SEL(0)));
4874 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4875 radeon_ring_write(ring, 0);
4876 radeon_ring_write(ring, VMID(0));
6f2043ce 4877
f96ab484
AD		 4878 /* HDP flush */
4879 /* We should be using the WAIT_REG_MEM packet here like in
4880 * cik_fence_ring_emit(), but it causes the CP to hang in this
4881 * context...
4882 */
4883 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4884 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4885 WRITE_DATA_DST_SEL(0)));
4886 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4887 radeon_ring_write(ring, 0);
4888 radeon_ring_write(ring, 0);
4889
4890 /* bits 0-15 are the VM contexts0-15 */
4891 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4892 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4893 WRITE_DATA_DST_SEL(0)));
4894 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4895 radeon_ring_write(ring, 0);
4896 radeon_ring_write(ring, 1 << vm->id);
4897
b07fdd38
AD		 4898 /* compute doesn't have PFP */
4899 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
4900 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4901 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4902 radeon_ring_write(ring, 0x0);
4903 }
cc066715 4904}
6f2043ce 4905
cc066715 4906/**
d0e092d9 4907 * cik_vm_set_page - update the page tables using sDMA
cc066715
AD		 4908 *
4909 * @rdev: radeon_device pointer
d0e092d9
AD		 4910 * @ib: indirect buffer to fill with commands
4911 * @pe: addr of the page entry
4912 * @addr: dst addr to write into pe
4913 * @count: number of page entries to update
4914 * @incr: increase next addr by incr bytes
4915 * @flags: access flags
cc066715 4916 *
d0e092d9 4917 * Update the page tables using CP or sDMA (CIK).
cc066715 4918 */
d0e092d9
AD		 4919void cik_vm_set_page(struct radeon_device *rdev,
4920 struct radeon_ib *ib,
4921 uint64_t pe,
4922 uint64_t addr, unsigned count,
4923 uint32_t incr, uint32_t flags)
4924{
4925 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4926 uint64_t value;
4927 unsigned ndw;
4928
4929 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4930 /* CP */
4931 while (count) {
4932 ndw = 2 + count * 2;
4933 if (ndw > 0x3FFE)
4934 ndw = 0x3FFE;
4935
4936 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4937 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4938 WRITE_DATA_DST_SEL(1));
4939 ib->ptr[ib->length_dw++] = pe;
4940 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4941 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4942 if (flags & RADEON_VM_PAGE_SYSTEM) {
4943 value = radeon_vm_map_gart(rdev, addr);
4944 value &= 0xFFFFFFFFFFFFF000ULL;
4945 } else if (flags & RADEON_VM_PAGE_VALID) {
4946 value = addr;
4947 } else {
4948 value = 0;
4949 }
4950 addr += incr;
4951 value |= r600_flags;
4952 ib->ptr[ib->length_dw++] = value;
4953 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4954 }
4955 }
4956 } else {
4957 /* DMA */
2483b4ea 4958 cik_sdma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
605de6b9 4959 }
605de6b9
AD		 4960}
4961
f6796cae
AD		 4962/*
4963 * RLC
4964 * The RLC is a multi-purpose microengine that handles a
4965 * variety of functions, the most important of which is
4966 * the interrupt controller.
4967 */
866d83de
AD		 4968static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
4969 bool enable)
f6796cae 4970{
866d83de 4971 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 4972
866d83de
AD		 4973 if (enable)
4974 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4975 else
4976 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 4977 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 4978}
f6796cae 4979
866d83de 4980static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 4981{
cc066715 4982 u32 tmp;
6f2043ce 4983
866d83de
AD		 4984 tmp = RREG32(RLC_LB_CNTL);
4985 if (enable)
4986 tmp |= LOAD_BALANCE_ENABLE;
4987 else
4988 tmp &= ~LOAD_BALANCE_ENABLE;
4989 WREG32(RLC_LB_CNTL, tmp);
4990}
cc066715 4991
866d83de
AD		 4992static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
4993{
4994 u32 i, j, k;
4995 u32 mask;
cc066715 4996
f6796cae
AD		 4997 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
4998 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
4999 cik_select_se_sh(rdev, i, j);
5000 for (k = 0; k < rdev->usec_timeout; k++) {
5001 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5002 break;
5003 udelay(1);
5004 }
5005 }
5006 }
5007 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
cc066715 5008
f6796cae
AD		 5009 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5010 for (k = 0; k < rdev->usec_timeout; k++) {
5011 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5012 break;
5013 udelay(1);
5014 }
5015}
cc066715 5016
22c775ce
AD		 5017static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5018{
5019 u32 tmp;
cc066715 5020
22c775ce
AD		 5021 tmp = RREG32(RLC_CNTL);
5022 if (tmp != rlc)
5023 WREG32(RLC_CNTL, rlc);
5024}
cc066715 5025
22c775ce
AD		 5026static u32 cik_halt_rlc(struct radeon_device *rdev)
5027{
5028 u32 data, orig;
cc066715 5029
22c775ce 5030 orig = data = RREG32(RLC_CNTL);
cc066715 5031
22c775ce
AD		 5032 if (data & RLC_ENABLE) {
5033 u32 i;
cc066715 5034
22c775ce
AD		 5035 data &= ~RLC_ENABLE;
5036 WREG32(RLC_CNTL, data);
cc066715 5037
22c775ce
AD		 5038 for (i = 0; i < rdev->usec_timeout; i++) {
5039 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5040 break;
5041 udelay(1);
5042 }
cc066715 5043
22c775ce
AD		 5044 cik_wait_for_rlc_serdes(rdev);
5045 }
cc066715 5046
22c775ce
AD		 5047 return orig;
5048}
cc066715 5049
a412fce0
AD		 5050void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5051{
5052 u32 tmp, i, mask;
5053
5054 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5055 WREG32(RLC_GPR_REG2, tmp);
5056
5057 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5058 for (i = 0; i < rdev->usec_timeout; i++) {
5059 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5060 break;
5061 udelay(1);
5062 }
5063
5064 for (i = 0; i < rdev->usec_timeout; i++) {
5065 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5066 break;
5067 udelay(1);
5068 }
5069}
5070
5071void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5072{
5073 u32 tmp;
5074
5075 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5076 WREG32(RLC_GPR_REG2, tmp);
5077}
5078
866d83de
AD		 5079/**
5080 * cik_rlc_stop - stop the RLC ME
5081 *
5082 * @rdev: radeon_device pointer
5083 *
5084 * Halt the RLC ME (MicroEngine) (CIK).
5085 */
5086static void cik_rlc_stop(struct radeon_device *rdev)
5087{
22c775ce 5088 WREG32(RLC_CNTL, 0);
866d83de
AD		 5089
5090 cik_enable_gui_idle_interrupt(rdev, false);
5091
866d83de
AD		 5092 cik_wait_for_rlc_serdes(rdev);
5093}
5094
f6796cae
AD		 5095/**
5096 * cik_rlc_start - start the RLC ME
5097 *
5098 * @rdev: radeon_device pointer
5099 *
5100 * Unhalt the RLC ME (MicroEngine) (CIK).
5101 */
5102static void cik_rlc_start(struct radeon_device *rdev)
5103{
f6796cae 5104 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 5105
866d83de 5106 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5107
f6796cae 5108 udelay(50);
6f2043ce
AD		 5109}
5110
5111/**
f6796cae 5112 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 5113 *
5114 * @rdev: radeon_device pointer
5115 *
f6796cae
AD		 5116 * Initialize the RLC registers, load the ucode,
5117 * and start the RLC (CIK).
5118 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 5119 */
f6796cae 5120static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 5121{
22c775ce 5122 u32 i, size, tmp;
f6796cae 5123 const __be32 *fw_data;
cc066715 5124
f6796cae
AD		 5125 if (!rdev->rlc_fw)
5126 return -EINVAL;
cc066715 5127
f6796cae
AD		 5128 switch (rdev->family) {
5129 case CHIP_BONAIRE:
5130 default:
5131 size = BONAIRE_RLC_UCODE_SIZE;
5132 break;
5133 case CHIP_KAVERI:
5134 size = KV_RLC_UCODE_SIZE;
5135 break;
5136 case CHIP_KABINI:
5137 size = KB_RLC_UCODE_SIZE;
5138 break;
5139 }
cc066715 5140
cc066715
AD		 5141 cik_rlc_stop(rdev);
5142
22c775ce
AD		 5143 /* disable CG */
5144 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5145 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 5146
866d83de 5147 si_rlc_reset(rdev);
6f2043ce 5148
22c775ce 5149 cik_init_pg(rdev);
6f2043ce 5150
22c775ce 5151 cik_init_cg(rdev);
cc066715 5152
f6796cae
AD		 5153 WREG32(RLC_LB_CNTR_INIT, 0);
5154 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 5155
f6796cae
AD		 5156 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5157 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5158 WREG32(RLC_LB_PARAMS, 0x00600408);
5159 WREG32(RLC_LB_CNTL, 0x80000004);
cc066715 5160
f6796cae
AD		 5161 WREG32(RLC_MC_CNTL, 0);
5162 WREG32(RLC_UCODE_CNTL, 0);
cc066715 5163
f6796cae
AD		 5164 fw_data = (const __be32 *)rdev->rlc_fw->data;
5165 WREG32(RLC_GPM_UCODE_ADDR, 0);
5166 for (i = 0; i < size; i++)
5167 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5168 WREG32(RLC_GPM_UCODE_ADDR, 0);
cc066715 5169
866d83de
AD		 5170 /* XXX - find out what chips support lbpw */
5171 cik_enable_lbpw(rdev, false);
cc066715 5172
22c775ce
AD		 5173 if (rdev->family == CHIP_BONAIRE)
5174 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 5175
f6796cae 5176 cik_rlc_start(rdev);
cc066715 5177
f6796cae
AD		 5178 return 0;
5179}
cc066715 5180
22c775ce
AD		 5181static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5182{
5183 u32 data, orig, tmp, tmp2;
cc066715 5184
22c775ce 5185 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 5186
473359bc 5187 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 5188 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5189
22c775ce 5190 tmp = cik_halt_rlc(rdev);
cc066715 5191
22c775ce
AD		 5192 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5193 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5194 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5195 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5196 WREG32(RLC_SERDES_WR_CTRL, tmp2);
cc066715 5197
22c775ce 5198 cik_update_rlc(rdev, tmp);
cc066715 5199
22c775ce
AD		 5200 data |= CGCG_EN | CGLS_EN;
5201 } else {
ddc76ff6 5202 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 5203
22c775ce
AD		 5204 RREG32(CB_CGTT_SCLK_CTRL);
5205 RREG32(CB_CGTT_SCLK_CTRL);
5206 RREG32(CB_CGTT_SCLK_CTRL);
5207 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 5208
22c775ce 5209 data &= ~(CGCG_EN | CGLS_EN);
cc066715 5210 }
6f2043ce 5211
22c775ce
AD		 5212 if (orig != data)
5213 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 5214
6f2043ce
AD		 5215}
5216
22c775ce 5217static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 5218{
22c775ce
AD		 5219 u32 data, orig, tmp = 0;
5220
473359bc
AD		 5221 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5222 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5223 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5224 orig = data = RREG32(CP_MEM_SLP_CNTL);
5225 data |= CP_MEM_LS_EN;
5226 if (orig != data)
5227 WREG32(CP_MEM_SLP_CNTL, data);
5228 }
5229 }
cc066715 5230
22c775ce
AD		 5231 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5232 data &= 0xfffffffd;
5233 if (orig != data)
5234 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5235
5236 tmp = cik_halt_rlc(rdev);
5237
5238 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5239 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5240 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5241 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5242 WREG32(RLC_SERDES_WR_CTRL, data);
5243
5244 cik_update_rlc(rdev, tmp);
5245
473359bc
AD		 5246 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5247 orig = data = RREG32(CGTS_SM_CTRL_REG);
5248 data &= ~SM_MODE_MASK;
5249 data |= SM_MODE(0x2);
5250 data |= SM_MODE_ENABLE;
5251 data &= ~CGTS_OVERRIDE;
5252 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5253 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5254 data &= ~CGTS_LS_OVERRIDE;
5255 data &= ~ON_MONITOR_ADD_MASK;
5256 data |= ON_MONITOR_ADD_EN;
5257 data |= ON_MONITOR_ADD(0x96);
5258 if (orig != data)
5259 WREG32(CGTS_SM_CTRL_REG, data);
5260 }
22c775ce
AD		 5261 } else {
5262 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5263 data |= 0x00000002;
5264 if (orig != data)
5265 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5266
5267 data = RREG32(RLC_MEM_SLP_CNTL);
5268 if (data & RLC_MEM_LS_EN) {
5269 data &= ~RLC_MEM_LS_EN;
5270 WREG32(RLC_MEM_SLP_CNTL, data);
5271 }
6f2043ce 5272
22c775ce
AD		 5273 data = RREG32(CP_MEM_SLP_CNTL);
5274 if (data & CP_MEM_LS_EN) {
5275 data &= ~CP_MEM_LS_EN;
5276 WREG32(CP_MEM_SLP_CNTL, data);
5277 }
cc066715 5278
22c775ce
AD		 5279 orig = data = RREG32(CGTS_SM_CTRL_REG);
5280 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5281 if (orig != data)
5282 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 5283
22c775ce 5284 tmp = cik_halt_rlc(rdev);
cc066715 5285
22c775ce
AD		 5286 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5287 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5288 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5289 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5290 WREG32(RLC_SERDES_WR_CTRL, data);
cc066715 5291
22c775ce 5292 cik_update_rlc(rdev, tmp);
cc066715 5293 }
6f2043ce 5294}
1c49165d 5295
22c775ce 5296static const u32 mc_cg_registers[] =
21a93e13 5297{
22c775ce
AD		 5298 MC_HUB_MISC_HUB_CG,
5299 MC_HUB_MISC_SIP_CG,
5300 MC_HUB_MISC_VM_CG,
5301 MC_XPB_CLK_GAT,
5302 ATC_MISC_CG,
5303 MC_CITF_MISC_WR_CG,
5304 MC_CITF_MISC_RD_CG,
5305 MC_CITF_MISC_VM_CG,
5306 VM_L2_CG,
5307};
21a93e13 5308
22c775ce
AD		 5309static void cik_enable_mc_ls(struct radeon_device *rdev,
5310 bool enable)
1c49165d 5311{
22c775ce
AD		 5312 int i;
5313 u32 orig, data;
1c49165d 5314
22c775ce
AD		 5315 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5316 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5317 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 5318 data |= MC_LS_ENABLE;
5319 else
5320 data &= ~MC_LS_ENABLE;
5321 if (data != orig)
5322 WREG32(mc_cg_registers[i], data);
1c49165d 5323 }
22c775ce 5324}
1c49165d 5325
22c775ce
AD		 5326static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5327 bool enable)
5328{
5329 int i;
5330 u32 orig, data;
5331
5332 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5333 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5334 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 5335 data |= MC_CG_ENABLE;
5336 else
5337 data &= ~MC_CG_ENABLE;
5338 if (data != orig)
5339 WREG32(mc_cg_registers[i], data);
1c49165d 5340 }
1c49165d
AD		 5341}
5342
22c775ce
AD		 5343static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5344 bool enable)
1c49165d 5345{
22c775ce 5346 u32 orig, data;
1c49165d 5347
473359bc 5348 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 5349 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5350 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 5351 } else {
22c775ce
AD		 5352 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5353 data |= 0xff000000;
5354 if (data != orig)
5355 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 5356
22c775ce
AD		 5357 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5358 data |= 0xff000000;
5359 if (data != orig)
5360 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5361 }
1c49165d
AD		 5362}
5363
22c775ce
AD		 5364static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5365 bool enable)
1c49165d 5366{
22c775ce
AD		 5367 u32 orig, data;
5368
473359bc 5369 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 5370 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5371 data |= 0x100;
5372 if (orig != data)
5373 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5374
5375 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5376 data |= 0x100;
5377 if (orig != data)
5378 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5379 } else {
5380 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5381 data &= ~0x100;
5382 if (orig != data)
5383 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 5384
22c775ce
AD		 5385 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5386 data &= ~0x100;
5387 if (orig != data)
5388 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5389 }
1c49165d
AD		 5390}
5391
22c775ce
AD		 5392static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5393 bool enable)
1c49165d 5394{
22c775ce 5395 u32 orig, data;
1c49165d 5396
473359bc 5397 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 5398 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5399 data = 0xfff;
5400 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 5401
22c775ce
AD		 5402 orig = data = RREG32(UVD_CGC_CTRL);
5403 data |= DCM;
5404 if (orig != data)
5405 WREG32(UVD_CGC_CTRL, data);
5406 } else {
5407 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5408 data &= ~0xfff;
5409 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 5410
22c775ce
AD		 5411 orig = data = RREG32(UVD_CGC_CTRL);
5412 data &= ~DCM;
5413 if (orig != data)
5414 WREG32(UVD_CGC_CTRL, data);
1c49165d 5415 }
22c775ce 5416}
1c49165d 5417
473359bc
AD		 5418static void cik_enable_bif_mgls(struct radeon_device *rdev,
5419 bool enable)
5420{
5421 u32 orig, data;
1c49165d 5422
473359bc 5423 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 5424
473359bc
AD		 5425 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5426 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5427 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5428 else
5429 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5430 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 5431
473359bc
AD		 5432 if (orig != data)
5433 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5434}
1c49165d 5435
22c775ce
AD		 5436static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5437 bool enable)
5438{
5439 u32 orig, data;
1c49165d 5440
22c775ce 5441 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 5442
473359bc 5443 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 5444 data &= ~CLOCK_GATING_DIS;
5445 else
5446 data |= CLOCK_GATING_DIS;
5447
5448 if (orig != data)
5449 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 5450}
5451
22c775ce
AD		 5452static void cik_enable_hdp_ls(struct radeon_device *rdev,
5453 bool enable)
1c49165d 5454{
22c775ce
AD		 5455 u32 orig, data;
5456
5457 orig = data = RREG32(HDP_MEM_POWER_LS);
5458
473359bc 5459 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 5460 data |= HDP_LS_ENABLE;
5461 else
5462 data &= ~HDP_LS_ENABLE;
5463
5464 if (orig != data)
5465 WREG32(HDP_MEM_POWER_LS, data);
5466}
5467
5468void cik_update_cg(struct radeon_device *rdev,
5469 u32 block, bool enable)
5470{
4214faf6 5471
22c775ce 5472 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 5473 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 5474 /* order matters! */
5475 if (enable) {
5476 cik_enable_mgcg(rdev, true);
5477 cik_enable_cgcg(rdev, true);
5478 } else {
5479 cik_enable_cgcg(rdev, false);
5480 cik_enable_mgcg(rdev, false);
5481 }
4214faf6 5482 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 5483 }
5484
5485 if (block & RADEON_CG_BLOCK_MC) {
5486 if (!(rdev->flags & RADEON_IS_IGP)) {
5487 cik_enable_mc_mgcg(rdev, enable);
5488 cik_enable_mc_ls(rdev, enable);
5489 }
5490 }
5491
5492 if (block & RADEON_CG_BLOCK_SDMA) {
5493 cik_enable_sdma_mgcg(rdev, enable);
5494 cik_enable_sdma_mgls(rdev, enable);
5495 }
5496
473359bc
AD		 5497 if (block & RADEON_CG_BLOCK_BIF) {
5498 cik_enable_bif_mgls(rdev, enable);
5499 }
5500
22c775ce
AD		 5501 if (block & RADEON_CG_BLOCK_UVD) {
5502 if (rdev->has_uvd)
5503 cik_enable_uvd_mgcg(rdev, enable);
5504 }
5505
5506 if (block & RADEON_CG_BLOCK_HDP) {
5507 cik_enable_hdp_mgcg(rdev, enable);
5508 cik_enable_hdp_ls(rdev, enable);
5509 }
1c49165d
AD		 5510}
5511
22c775ce 5512static void cik_init_cg(struct radeon_device *rdev)
1c49165d 5513{
22c775ce 5514
ddc76ff6 5515 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 5516
5517 if (rdev->has_uvd)
5518 si_init_uvd_internal_cg(rdev);
5519
5520 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5521 RADEON_CG_BLOCK_SDMA |
473359bc 5522 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 5523 RADEON_CG_BLOCK_UVD |
5524 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 5525}
5526
473359bc 5527static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 5528{
473359bc
AD		 5529 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5530 RADEON_CG_BLOCK_SDMA |
5531 RADEON_CG_BLOCK_BIF |
5532 RADEON_CG_BLOCK_UVD |
5533 RADEON_CG_BLOCK_HDP), false);
5534
5535 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 5536}
5537
22c775ce
AD		 5538static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5539 bool enable)
1c49165d 5540{
22c775ce 5541 u32 data, orig;
1c49165d 5542
22c775ce 5543 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5544 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 5545 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5546 else
5547 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5548 if (orig != data)
5549 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 5550}
5551
22c775ce
AD		 5552static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5553 bool enable)
1c49165d 5554{
22c775ce
AD		 5555 u32 data, orig;
5556
5557 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5558 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 5559 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5560 else
5561 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5562 if (orig != data)
5563 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 5564}
5565
22c775ce 5566static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 5567{
22c775ce 5568 u32 data, orig;
3ec7d11b 5569
22c775ce 5570 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5571 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 5572 data &= ~DISABLE_CP_PG;
5573 else
5574 data |= DISABLE_CP_PG;
5575 if (orig != data)
5576 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 5577}
5578
22c775ce 5579static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 5580{
22c775ce 5581 u32 data, orig;
f96ab484 5582
22c775ce 5583 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5584 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 5585 data &= ~DISABLE_GDS_PG;
5586 else
5587 data |= DISABLE_GDS_PG;
5588 if (orig != data)
5589 WREG32(RLC_PG_CNTL, data);
5590}
5591
5592#define CP_ME_TABLE_SIZE 96
5593#define CP_ME_TABLE_OFFSET 2048
5594#define CP_MEC_TABLE_OFFSET 4096
5595
5596void cik_init_cp_pg_table(struct radeon_device *rdev)
5597{
5598 const __be32 *fw_data;
5599 volatile u32 *dst_ptr;
5600 int me, i, max_me = 4;
5601 u32 bo_offset = 0;
5602 u32 table_offset;
5603
5604 if (rdev->family == CHIP_KAVERI)
5605 max_me = 5;
5606
5607 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 5608 return;
5609
22c775ce
AD		 5610 /* write the cp table buffer */
5611 dst_ptr = rdev->rlc.cp_table_ptr;
5612 for (me = 0; me < max_me; me++) {
5613 if (me == 0) {
5614 fw_data = (const __be32 *)rdev->ce_fw->data;
5615 table_offset = CP_ME_TABLE_OFFSET;
5616 } else if (me == 1) {
5617 fw_data = (const __be32 *)rdev->pfp_fw->data;
5618 table_offset = CP_ME_TABLE_OFFSET;
5619 } else if (me == 2) {
5620 fw_data = (const __be32 *)rdev->me_fw->data;
5621 table_offset = CP_ME_TABLE_OFFSET;
5622 } else {
5623 fw_data = (const __be32 *)rdev->mec_fw->data;
5624 table_offset = CP_MEC_TABLE_OFFSET;
5625 }
5626
5627 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
5628 dst_ptr[bo_offset + i] = be32_to_cpu(fw_data[table_offset + i]);
5629 }
5630 bo_offset += CP_ME_TABLE_SIZE;
f96ab484 5631 }
22c775ce 5632}
f96ab484 5633
22c775ce
AD		 5634static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
5635 bool enable)
5636{
5637 u32 data, orig;
5638
2b19d17f 5639 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 5640 orig = data = RREG32(RLC_PG_CNTL);
5641 data |= GFX_PG_ENABLE;
5642 if (orig != data)
5643 WREG32(RLC_PG_CNTL, data);
5644
5645 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5646 data |= AUTO_PG_EN;
5647 if (orig != data)
5648 WREG32(RLC_AUTO_PG_CTRL, data);
5649 } else {
5650 orig = data = RREG32(RLC_PG_CNTL);
5651 data &= ~GFX_PG_ENABLE;
5652 if (orig != data)
5653 WREG32(RLC_PG_CNTL, data);
f96ab484 5654
22c775ce
AD		 5655 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5656 data &= ~AUTO_PG_EN;
5657 if (orig != data)
5658 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 5659
22c775ce
AD		 5660 data = RREG32(DB_RENDER_CONTROL);
5661 }
5662}
f96ab484 5663
22c775ce
AD		 5664static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5665{
5666 u32 mask = 0, tmp, tmp1;
5667 int i;
f96ab484 5668
22c775ce
AD		 5669 cik_select_se_sh(rdev, se, sh);
5670 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5671 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5672 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
f96ab484 5673
22c775ce 5674 tmp &= 0xffff0000;
f96ab484 5675
22c775ce
AD		 5676 tmp |= tmp1;
5677 tmp >>= 16;
5678
5679 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
5680 mask <<= 1;
5681 mask |= 1;
b07fdd38 5682 }
22c775ce
AD		 5683
5684 return (~tmp) & mask;
f96ab484
AD		 5685}
5686
22c775ce 5687static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 5688{
22c775ce
AD		 5689 u32 i, j, k, active_cu_number = 0;
5690 u32 mask, counter, cu_bitmap;
5691 u32 tmp = 0;
d0e092d9 5692
22c775ce
AD		 5693 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5694 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5695 mask = 1;
5696 cu_bitmap = 0;
5697 counter = 0;
5698 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
5699 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
5700 if (counter < 2)
5701 cu_bitmap |= mask;
5702 counter ++;
d0e092d9 5703 }
22c775ce 5704 mask <<= 1;
d0e092d9 5705 }
d0e092d9 5706
22c775ce
AD		 5707 active_cu_number += counter;
5708 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 5709 }
d0e092d9 5710 }
22c775ce
AD		 5711
5712 WREG32(RLC_PG_AO_CU_MASK, tmp);
5713
5714 tmp = RREG32(RLC_MAX_PG_CU);
5715 tmp &= ~MAX_PU_CU_MASK;
5716 tmp |= MAX_PU_CU(active_cu_number);
5717 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 5718}
5719
22c775ce
AD		 5720static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
5721 bool enable)
605de6b9 5722{
22c775ce 5723 u32 data, orig;
605de6b9 5724
22c775ce 5725 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5726 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 5727 data |= STATIC_PER_CU_PG_ENABLE;
5728 else
5729 data &= ~STATIC_PER_CU_PG_ENABLE;
5730 if (orig != data)
5731 WREG32(RLC_PG_CNTL, data);
5732}
5733
5734static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
5735 bool enable)
5736{
5737 u32 data, orig;
605de6b9 5738
22c775ce 5739 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5740 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 5741 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 5742 else
22c775ce
AD		 5743 data &= ~DYN_PER_CU_PG_ENABLE;
5744 if (orig != data)
5745 WREG32(RLC_PG_CNTL, data);
5746}
605de6b9 5747
22c775ce
AD		 5748#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
5749#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
5750
5751static void cik_init_gfx_cgpg(struct radeon_device *rdev)
5752{
5753 u32 data, orig;
5754 u32 i;
5755
5756 if (rdev->rlc.cs_data) {
5757 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5758 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 5759 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 5760 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 5761 } else {
22c775ce
AD		 5762 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5763 for (i = 0; i < 3; i++)
5764 WREG32(RLC_GPM_SCRATCH_DATA, 0);
5765 }
5766 if (rdev->rlc.reg_list) {
5767 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
5768 for (i = 0; i < rdev->rlc.reg_list_size; i++)
5769 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 5770 }
605de6b9 5771
22c775ce
AD		 5772 orig = data = RREG32(RLC_PG_CNTL);
5773 data |= GFX_PG_SRC;
5774 if (orig != data)
5775 WREG32(RLC_PG_CNTL, data);
605de6b9 5776
22c775ce
AD		 5777 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5778 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 5779
22c775ce
AD		 5780 data = RREG32(CP_RB_WPTR_POLL_CNTL);
5781 data &= ~IDLE_POLL_COUNT_MASK;
5782 data |= IDLE_POLL_COUNT(0x60);
5783 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 5784
22c775ce
AD		 5785 data = 0x10101010;
5786 WREG32(RLC_PG_DELAY, data);
605de6b9 5787
22c775ce
AD		 5788 data = RREG32(RLC_PG_DELAY_2);
5789 data &= ~0xff;
5790 data |= 0x3;
5791 WREG32(RLC_PG_DELAY_2, data);
605de6b9 5792
22c775ce
AD		 5793 data = RREG32(RLC_AUTO_PG_CTRL);
5794 data &= ~GRBM_REG_SGIT_MASK;
5795 data |= GRBM_REG_SGIT(0x700);
5796 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 5797
605de6b9
AD		 5798}
5799
22c775ce 5800static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 5801{
473359bc
AD		 5802 cik_enable_gfx_cgpg(rdev, enable);
5803 cik_enable_gfx_static_mgpg(rdev, enable);
5804 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 5805}
f6796cae 5806
a0f38609
AD		 5807u32 cik_get_csb_size(struct radeon_device *rdev)
5808{
5809 u32 count = 0;
5810 const struct cs_section_def *sect = NULL;
5811 const struct cs_extent_def *ext = NULL;
f6796cae 5812
a0f38609
AD		 5813 if (rdev->rlc.cs_data == NULL)
5814 return 0;
f6796cae 5815
a0f38609
AD		 5816 /* begin clear state */
5817 count += 2;
5818 /* context control state */
5819 count += 3;
5820
5821 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5822 for (ext = sect->section; ext->extent != NULL; ++ext) {
5823 if (sect->id == SECT_CONTEXT)
5824 count += 2 + ext->reg_count;
5825 else
5826 return 0;
f6796cae
AD		 5827 }
5828 }
a0f38609
AD		 5829 /* pa_sc_raster_config/pa_sc_raster_config1 */
5830 count += 4;
5831 /* end clear state */
5832 count += 2;
5833 /* clear state */
5834 count += 2;
f6796cae 5835
a0f38609 5836 return count;
f6796cae
AD		 5837}
5838
a0f38609 5839void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 5840{
a0f38609
AD		 5841 u32 count = 0, i;
5842 const struct cs_section_def *sect = NULL;
5843 const struct cs_extent_def *ext = NULL;
f6796cae 5844
a0f38609
AD		 5845 if (rdev->rlc.cs_data == NULL)
5846 return;
5847 if (buffer == NULL)
5848 return;
f6796cae 5849
a0f38609
AD		 5850 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5851 buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5852
5853 buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5854 buffer[count++] = 0x80000000;
5855 buffer[count++] = 0x80000000;
5856
5857 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5858 for (ext = sect->section; ext->extent != NULL; ++ext) {
5859 if (sect->id == SECT_CONTEXT) {
5860 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5861 buffer[count++] = ext->reg_index - 0xa000;
5862 for (i = 0; i < ext->reg_count; i++)
5863 buffer[count++] = ext->extent[i];
5864 } else {
5865 return;
5866 }
5867 }
5868 }
f6796cae 5869
a0f38609
AD		 5870 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 2);
5871 buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
f6796cae
AD		 5872 switch (rdev->family) {
5873 case CHIP_BONAIRE:
a0f38609
AD		 5874 buffer[count++] = 0x16000012;
5875 buffer[count++] = 0x00000000;
f6796cae
AD		 5876 break;
5877 case CHIP_KAVERI:
a0f38609
AD		 5878 buffer[count++] = 0x00000000; /* XXX */
5879 buffer[count++] = 0x00000000;
f6796cae
AD		 5880 break;
5881 case CHIP_KABINI:
a0f38609
AD		 5882 buffer[count++] = 0x00000000; /* XXX */
5883 buffer[count++] = 0x00000000;
5884 break;
5885 default:
5886 buffer[count++] = 0x00000000;
5887 buffer[count++] = 0x00000000;
f6796cae
AD		 5888 break;
5889 }
5890
a0f38609
AD		 5891 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5892 buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
f6796cae 5893
a0f38609
AD		 5894 buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5895 buffer[count++] = 0;
5896}
f6796cae 5897
473359bc 5898static void cik_init_pg(struct radeon_device *rdev)
22c775ce 5899{
473359bc 5900 if (rdev->pg_flags) {
22c775ce
AD		 5901 cik_enable_sck_slowdown_on_pu(rdev, true);
5902 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 5903 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 5904 cik_init_gfx_cgpg(rdev);
5905 cik_enable_cp_pg(rdev, true);
5906 cik_enable_gds_pg(rdev, true);
5907 }
22c775ce
AD		 5908 cik_init_ao_cu_mask(rdev);
5909 cik_update_gfx_pg(rdev, true);
5910 }
5911}
f6796cae 5912
473359bc
AD		 5913static void cik_fini_pg(struct radeon_device *rdev)
5914{
5915 if (rdev->pg_flags) {
5916 cik_update_gfx_pg(rdev, false);
2b19d17f 5917 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 5918 cik_enable_cp_pg(rdev, false);
5919 cik_enable_gds_pg(rdev, false);
5920 }
5921 }
f6796cae 5922}
a59781bb
AD		 5923
5924/*
5925 * Interrupts
5926 * Starting with r6xx, interrupts are handled via a ring buffer.
5927 * Ring buffers are areas of GPU accessible memory that the GPU
5928 * writes interrupt vectors into and the host reads vectors out of.
5929 * There is a rptr (read pointer) that determines where the
5930 * host is currently reading, and a wptr (write pointer)
5931 * which determines where the GPU has written. When the
5932 * pointers are equal, the ring is idle. When the GPU
5933 * writes vectors to the ring buffer, it increments the
5934 * wptr. When there is an interrupt, the host then starts
5935 * fetching commands and processing them until the pointers are
5936 * equal again at which point it updates the rptr.
5937 */
5938
5939/**
5940 * cik_enable_interrupts - Enable the interrupt ring buffer
5941 *
5942 * @rdev: radeon_device pointer
5943 *
5944 * Enable the interrupt ring buffer (CIK).
5945 */
5946static void cik_enable_interrupts(struct radeon_device *rdev)
5947{
5948 u32 ih_cntl = RREG32(IH_CNTL);
5949 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5950
5951 ih_cntl |= ENABLE_INTR;
5952 ih_rb_cntl |= IH_RB_ENABLE;
5953 WREG32(IH_CNTL, ih_cntl);
5954 WREG32(IH_RB_CNTL, ih_rb_cntl);
5955 rdev->ih.enabled = true;
5956}
5957
5958/**
5959 * cik_disable_interrupts - Disable the interrupt ring buffer
5960 *
5961 * @rdev: radeon_device pointer
5962 *
5963 * Disable the interrupt ring buffer (CIK).
5964 */
5965static void cik_disable_interrupts(struct radeon_device *rdev)
5966{
5967 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5968 u32 ih_cntl = RREG32(IH_CNTL);
5969
5970 ih_rb_cntl &= ~IH_RB_ENABLE;
5971 ih_cntl &= ~ENABLE_INTR;
5972 WREG32(IH_RB_CNTL, ih_rb_cntl);
5973 WREG32(IH_CNTL, ih_cntl);
5974 /* set rptr, wptr to 0 */
5975 WREG32(IH_RB_RPTR, 0);
5976 WREG32(IH_RB_WPTR, 0);
5977 rdev->ih.enabled = false;
5978 rdev->ih.rptr = 0;
5979}
5980
5981/**
5982 * cik_disable_interrupt_state - Disable all interrupt sources
5983 *
5984 * @rdev: radeon_device pointer
5985 *
5986 * Clear all interrupt enable bits used by the driver (CIK).
5987 */
5988static void cik_disable_interrupt_state(struct radeon_device *rdev)
5989{
5990 u32 tmp;
5991
5992 /* gfx ring */
4214faf6
AD		 5993 tmp = RREG32(CP_INT_CNTL_RING0) &
5994 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5995 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 5996 /* sdma */
5997 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5998 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5999 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6000 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 6001 /* compute queues */
6002 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6003 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6004 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6005 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6006 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6007 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6008 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6009 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6010 /* grbm */
6011 WREG32(GRBM_INT_CNTL, 0);
6012 /* vline/vblank, etc. */
6013 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6014 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6015 if (rdev->num_crtc >= 4) {
6016 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6017 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6018 }
6019 if (rdev->num_crtc >= 6) {
6020 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6021 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6022 }
6023
6024 /* dac hotplug */
6025 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6026
6027 /* digital hotplug */
6028 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6029 WREG32(DC_HPD1_INT_CONTROL, tmp);
6030 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6031 WREG32(DC_HPD2_INT_CONTROL, tmp);
6032 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6033 WREG32(DC_HPD3_INT_CONTROL, tmp);
6034 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6035 WREG32(DC_HPD4_INT_CONTROL, tmp);
6036 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6037 WREG32(DC_HPD5_INT_CONTROL, tmp);
6038 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6039 WREG32(DC_HPD6_INT_CONTROL, tmp);
6040
6041}
6042
6043/**
6044 * cik_irq_init - init and enable the interrupt ring
6045 *
6046 * @rdev: radeon_device pointer
6047 *
6048 * Allocate a ring buffer for the interrupt controller,
6049 * enable the RLC, disable interrupts, enable the IH
6050 * ring buffer and enable it (CIK).
6051 * Called at device load and reume.
6052 * Returns 0 for success, errors for failure.
6053 */
6054static int cik_irq_init(struct radeon_device *rdev)
6055{
6056 int ret = 0;
6057 int rb_bufsz;
6058 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6059
6060 /* allocate ring */
6061 ret = r600_ih_ring_alloc(rdev);
6062 if (ret)
6063 return ret;
6064
6065 /* disable irqs */
6066 cik_disable_interrupts(rdev);
6067
6068 /* init rlc */
6069 ret = cik_rlc_resume(rdev);
6070 if (ret) {
6071 r600_ih_ring_fini(rdev);
6072 return ret;
6073 }
6074
6075 /* setup interrupt control */
6076 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6077 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6078 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6079 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6080 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6081 */
6082 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6083 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6084 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6085 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6086
6087 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 6088 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 6089
6090 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6091 IH_WPTR_OVERFLOW_CLEAR |
6092 (rb_bufsz << 1));
6093
6094 if (rdev->wb.enabled)
6095 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6096
6097 /* set the writeback address whether it's enabled or not */
6098 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6099 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6100
6101 WREG32(IH_RB_CNTL, ih_rb_cntl);
6102
6103 /* set rptr, wptr to 0 */
6104 WREG32(IH_RB_RPTR, 0);
6105 WREG32(IH_RB_WPTR, 0);
6106
6107 /* Default settings for IH_CNTL (disabled at first) */
6108 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6109 /* RPTR_REARM only works if msi's are enabled */
6110 if (rdev->msi_enabled)
6111 ih_cntl |= RPTR_REARM;
6112 WREG32(IH_CNTL, ih_cntl);
6113
6114 /* force the active interrupt state to all disabled */
6115 cik_disable_interrupt_state(rdev);
6116
6117 pci_set_master(rdev->pdev);
6118
6119 /* enable irqs */
6120 cik_enable_interrupts(rdev);
6121
6122 return ret;
6123}
6124
6125/**
6126 * cik_irq_set - enable/disable interrupt sources
6127 *
6128 * @rdev: radeon_device pointer
6129 *
6130 * Enable interrupt sources on the GPU (vblanks, hpd,
6131 * etc.) (CIK).
6132 * Returns 0 for success, errors for failure.
6133 */
6134int cik_irq_set(struct radeon_device *rdev)
6135{
4214faf6 6136 u32 cp_int_cntl;
2b0781a6
AD		 6137 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6138 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
a59781bb
AD		 6139 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6140 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6141 u32 grbm_int_cntl = 0;
21a93e13 6142 u32 dma_cntl, dma_cntl1;
41a524ab 6143 u32 thermal_int;
a59781bb
AD		 6144
6145 if (!rdev->irq.installed) {
6146 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6147 return -EINVAL;
6148 }
6149 /* don't enable anything if the ih is disabled */
6150 if (!rdev->ih.enabled) {
6151 cik_disable_interrupts(rdev);
6152 /* force the active interrupt state to all disabled */
6153 cik_disable_interrupt_state(rdev);
6154 return 0;
6155 }
6156
4214faf6
AD		 6157 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6158 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6159 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6160
a59781bb
AD		 6161 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6162 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6163 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6164 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6165 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6166 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6167
21a93e13
AD		 6168 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6169 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6170
2b0781a6
AD		 6171 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6172 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6173 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6174 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6175 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6176 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6177 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6178 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6179
cc8dbbb4
AD		 6180 if (rdev->flags & RADEON_IS_IGP)
6181 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6182 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6183 else
6184 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6185 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
41a524ab 6186
a59781bb
AD		 6187 /* enable CP interrupts on all rings */
6188 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6189 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6190 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6191 }
2b0781a6
AD		 6192 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6193 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6194 DRM_DEBUG("si_irq_set: sw int cp1\n");
6195 if (ring->me == 1) {
6196 switch (ring->pipe) {
6197 case 0:
6198 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6199 break;
6200 case 1:
6201 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6202 break;
6203 case 2:
6204 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6205 break;
6206 case 3:
6207 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6208 break;
6209 default:
6210 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6211 break;
6212 }
6213 } else if (ring->me == 2) {
6214 switch (ring->pipe) {
6215 case 0:
6216 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6217 break;
6218 case 1:
6219 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6220 break;
6221 case 2:
6222 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6223 break;
6224 case 3:
6225 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6226 break;
6227 default:
6228 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6229 break;
6230 }
6231 } else {
6232 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6233 }
6234 }
6235 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6236 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6237 DRM_DEBUG("si_irq_set: sw int cp2\n");
6238 if (ring->me == 1) {
6239 switch (ring->pipe) {
6240 case 0:
6241 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6242 break;
6243 case 1:
6244 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6245 break;
6246 case 2:
6247 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6248 break;
6249 case 3:
6250 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6251 break;
6252 default:
6253 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6254 break;
6255 }
6256 } else if (ring->me == 2) {
6257 switch (ring->pipe) {
6258 case 0:
6259 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6260 break;
6261 case 1:
6262 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6263 break;
6264 case 2:
6265 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6266 break;
6267 case 3:
6268 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6269 break;
6270 default:
6271 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6272 break;
6273 }
6274 } else {
6275 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6276 }
6277 }
a59781bb 6278
21a93e13
AD		 6279 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6280 DRM_DEBUG("cik_irq_set: sw int dma\n");
6281 dma_cntl |= TRAP_ENABLE;
6282 }
6283
6284 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6285 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6286 dma_cntl1 |= TRAP_ENABLE;
6287 }
6288
a59781bb
AD		 6289 if (rdev->irq.crtc_vblank_int[0] ||
6290 atomic_read(&rdev->irq.pflip[0])) {
6291 DRM_DEBUG("cik_irq_set: vblank 0\n");
6292 crtc1 |= VBLANK_INTERRUPT_MASK;
6293 }
6294 if (rdev->irq.crtc_vblank_int[1] ||
6295 atomic_read(&rdev->irq.pflip[1])) {
6296 DRM_DEBUG("cik_irq_set: vblank 1\n");
6297 crtc2 |= VBLANK_INTERRUPT_MASK;
6298 }
6299 if (rdev->irq.crtc_vblank_int[2] ||
6300 atomic_read(&rdev->irq.pflip[2])) {
6301 DRM_DEBUG("cik_irq_set: vblank 2\n");
6302 crtc3 |= VBLANK_INTERRUPT_MASK;
6303 }
6304 if (rdev->irq.crtc_vblank_int[3] ||
6305 atomic_read(&rdev->irq.pflip[3])) {
6306 DRM_DEBUG("cik_irq_set: vblank 3\n");
6307 crtc4 |= VBLANK_INTERRUPT_MASK;
6308 }
6309 if (rdev->irq.crtc_vblank_int[4] ||
6310 atomic_read(&rdev->irq.pflip[4])) {
6311 DRM_DEBUG("cik_irq_set: vblank 4\n");
6312 crtc5 |= VBLANK_INTERRUPT_MASK;
6313 }
6314 if (rdev->irq.crtc_vblank_int[5] ||
6315 atomic_read(&rdev->irq.pflip[5])) {
6316 DRM_DEBUG("cik_irq_set: vblank 5\n");
6317 crtc6 |= VBLANK_INTERRUPT_MASK;
6318 }
6319 if (rdev->irq.hpd[0]) {
6320 DRM_DEBUG("cik_irq_set: hpd 1\n");
6321 hpd1 |= DC_HPDx_INT_EN;
6322 }
6323 if (rdev->irq.hpd[1]) {
6324 DRM_DEBUG("cik_irq_set: hpd 2\n");
6325 hpd2 |= DC_HPDx_INT_EN;
6326 }
6327 if (rdev->irq.hpd[2]) {
6328 DRM_DEBUG("cik_irq_set: hpd 3\n");
6329 hpd3 |= DC_HPDx_INT_EN;
6330 }
6331 if (rdev->irq.hpd[3]) {
6332 DRM_DEBUG("cik_irq_set: hpd 4\n");
6333 hpd4 |= DC_HPDx_INT_EN;
6334 }
6335 if (rdev->irq.hpd[4]) {
6336 DRM_DEBUG("cik_irq_set: hpd 5\n");
6337 hpd5 |= DC_HPDx_INT_EN;
6338 }
6339 if (rdev->irq.hpd[5]) {
6340 DRM_DEBUG("cik_irq_set: hpd 6\n");
6341 hpd6 |= DC_HPDx_INT_EN;
6342 }
6343
41a524ab
AD		 6344 if (rdev->irq.dpm_thermal) {
6345 DRM_DEBUG("dpm thermal\n");
cc8dbbb4
AD		 6346 if (rdev->flags & RADEON_IS_IGP)
6347 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6348 else
6349 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
41a524ab
AD		 6350 }
6351
a59781bb
AD		 6352 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6353
21a93e13
AD		 6354 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6355 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6356
2b0781a6
AD		 6357 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6358 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6359 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6360 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6361 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6362 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6363 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6364 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6365
a59781bb
AD		 6366 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6367
6368 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6369 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6370 if (rdev->num_crtc >= 4) {
6371 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6372 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6373 }
6374 if (rdev->num_crtc >= 6) {
6375 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6376 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6377 }
6378
6379 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6380 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6381 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6382 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6383 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6384 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6385
cc8dbbb4
AD		 6386 if (rdev->flags & RADEON_IS_IGP)
6387 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6388 else
6389 WREG32_SMC(CG_THERMAL_INT, thermal_int);
41a524ab 6390
a59781bb
AD		 6391 return 0;
6392}
6393
6394/**
6395 * cik_irq_ack - ack interrupt sources
6396 *
6397 * @rdev: radeon_device pointer
6398 *
6399 * Ack interrupt sources on the GPU (vblanks, hpd,
6400 * etc.) (CIK). Certain interrupts sources are sw
6401 * generated and do not require an explicit ack.
6402 */
6403static inline void cik_irq_ack(struct radeon_device *rdev)
6404{
6405 u32 tmp;
6406
6407 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6408 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6409 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6410 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6411 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6412 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6413 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6414
6415 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6416 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6417 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6418 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6419 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6420 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6421 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6422 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6423
6424 if (rdev->num_crtc >= 4) {
6425 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6426 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6427 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6428 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6429 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6430 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6431 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6432 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6433 }
6434
6435 if (rdev->num_crtc >= 6) {
6436 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6437 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6438 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6439 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6440 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6441 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6442 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6443 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6444 }
6445
6446 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6447 tmp = RREG32(DC_HPD1_INT_CONTROL);
6448 tmp |= DC_HPDx_INT_ACK;
6449 WREG32(DC_HPD1_INT_CONTROL, tmp);
6450 }
6451 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6452 tmp = RREG32(DC_HPD2_INT_CONTROL);
6453 tmp |= DC_HPDx_INT_ACK;
6454 WREG32(DC_HPD2_INT_CONTROL, tmp);
6455 }
6456 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6457 tmp = RREG32(DC_HPD3_INT_CONTROL);
6458 tmp |= DC_HPDx_INT_ACK;
6459 WREG32(DC_HPD3_INT_CONTROL, tmp);
6460 }
6461 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6462 tmp = RREG32(DC_HPD4_INT_CONTROL);
6463 tmp |= DC_HPDx_INT_ACK;
6464 WREG32(DC_HPD4_INT_CONTROL, tmp);
6465 }
6466 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6467 tmp = RREG32(DC_HPD5_INT_CONTROL);
6468 tmp |= DC_HPDx_INT_ACK;
6469 WREG32(DC_HPD5_INT_CONTROL, tmp);
6470 }
6471 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6472 tmp = RREG32(DC_HPD5_INT_CONTROL);
6473 tmp |= DC_HPDx_INT_ACK;
6474 WREG32(DC_HPD6_INT_CONTROL, tmp);
6475 }
6476}
6477
6478/**
6479 * cik_irq_disable - disable interrupts
6480 *
6481 * @rdev: radeon_device pointer
6482 *
6483 * Disable interrupts on the hw (CIK).
6484 */
6485static void cik_irq_disable(struct radeon_device *rdev)
6486{
6487 cik_disable_interrupts(rdev);
6488 /* Wait and acknowledge irq */
6489 mdelay(1);
6490 cik_irq_ack(rdev);
6491 cik_disable_interrupt_state(rdev);
6492}
6493
6494/**
6495 * cik_irq_disable - disable interrupts for suspend
6496 *
6497 * @rdev: radeon_device pointer
6498 *
6499 * Disable interrupts and stop the RLC (CIK).
6500 * Used for suspend.
6501 */
6502static void cik_irq_suspend(struct radeon_device *rdev)
6503{
6504 cik_irq_disable(rdev);
6505 cik_rlc_stop(rdev);
6506}
6507
6508/**
6509 * cik_irq_fini - tear down interrupt support
6510 *
6511 * @rdev: radeon_device pointer
6512 *
6513 * Disable interrupts on the hw and free the IH ring
6514 * buffer (CIK).
6515 * Used for driver unload.
6516 */
6517static void cik_irq_fini(struct radeon_device *rdev)
6518{
6519 cik_irq_suspend(rdev);
6520 r600_ih_ring_fini(rdev);
6521}
6522
6523/**
6524 * cik_get_ih_wptr - get the IH ring buffer wptr
6525 *
6526 * @rdev: radeon_device pointer
6527 *
6528 * Get the IH ring buffer wptr from either the register
6529 * or the writeback memory buffer (CIK). Also check for
6530 * ring buffer overflow and deal with it.
6531 * Used by cik_irq_process().
6532 * Returns the value of the wptr.
6533 */
6534static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6535{
6536 u32 wptr, tmp;
6537
6538 if (rdev->wb.enabled)
6539 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6540 else
6541 wptr = RREG32(IH_RB_WPTR);
6542
6543 if (wptr & RB_OVERFLOW) {
6544 /* When a ring buffer overflow happen start parsing interrupt
6545 * from the last not overwritten vector (wptr + 16). Hopefully
6546 * this should allow us to catchup.
6547 */
6548 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6549 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6550 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6551 tmp = RREG32(IH_RB_CNTL);
6552 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6553 WREG32(IH_RB_CNTL, tmp);
6554 }
6555 return (wptr & rdev->ih.ptr_mask);
6556}
6557
6558/* CIK IV Ring
6559 * Each IV ring entry is 128 bits:
6560 * [7:0] - interrupt source id
6561 * [31:8] - reserved
6562 * [59:32] - interrupt source data
6563 * [63:60] - reserved
21a93e13
AD		 6564 * [71:64] - RINGID
6565 * CP:
6566 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 6567 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
6568 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
6569 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
6570 * PIPE_ID - ME0 0=3D
6571 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 6572 * SDMA:
6573 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
6574 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
6575 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 6576 * [79:72] - VMID
6577 * [95:80] - PASID
6578 * [127:96] - reserved
6579 */
6580/**
6581 * cik_irq_process - interrupt handler
6582 *
6583 * @rdev: radeon_device pointer
6584 *
6585 * Interrupt hander (CIK). Walk the IH ring,
6586 * ack interrupts and schedule work to handle
6587 * interrupt events.
6588 * Returns irq process return code.
6589 */
6590int cik_irq_process(struct radeon_device *rdev)
6591{
2b0781a6
AD		 6592 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6593 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 6594 u32 wptr;
6595 u32 rptr;
6596 u32 src_id, src_data, ring_id;
6597 u8 me_id, pipe_id, queue_id;
6598 u32 ring_index;
6599 bool queue_hotplug = false;
6600 bool queue_reset = false;
3ec7d11b 6601 u32 addr, status, mc_client;
41a524ab 6602 bool queue_thermal = false;
a59781bb
AD		 6603
6604 if (!rdev->ih.enabled || rdev->shutdown)
6605 return IRQ_NONE;
6606
6607 wptr = cik_get_ih_wptr(rdev);
6608
6609restart_ih:
6610 /* is somebody else already processing irqs? */
6611 if (atomic_xchg(&rdev->ih.lock, 1))
6612 return IRQ_NONE;
6613
6614 rptr = rdev->ih.rptr;
6615 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6616
6617 /* Order reading of wptr vs. reading of IH ring data */
6618 rmb();
6619
6620 /* display interrupts */
6621 cik_irq_ack(rdev);
6622
6623 while (rptr != wptr) {
6624 /* wptr/rptr are in bytes! */
6625 ring_index = rptr / 4;
6626 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6627 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6628 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 6629
6630 switch (src_id) {
6631 case 1: /* D1 vblank/vline */
6632 switch (src_data) {
6633 case 0: /* D1 vblank */
6634 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
6635 if (rdev->irq.crtc_vblank_int[0]) {
6636 drm_handle_vblank(rdev->ddev, 0);
6637 rdev->pm.vblank_sync = true;
6638 wake_up(&rdev->irq.vblank_queue);
6639 }
6640 if (atomic_read(&rdev->irq.pflip[0]))
6641 radeon_crtc_handle_flip(rdev, 0);
6642 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6643 DRM_DEBUG("IH: D1 vblank\n");
6644 }
6645 break;
6646 case 1: /* D1 vline */
6647 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
6648 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6649 DRM_DEBUG("IH: D1 vline\n");
6650 }
6651 break;
6652 default:
6653 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6654 break;
6655 }
6656 break;
6657 case 2: /* D2 vblank/vline */
6658 switch (src_data) {
6659 case 0: /* D2 vblank */
6660 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6661 if (rdev->irq.crtc_vblank_int[1]) {
6662 drm_handle_vblank(rdev->ddev, 1);
6663 rdev->pm.vblank_sync = true;
6664 wake_up(&rdev->irq.vblank_queue);
6665 }
6666 if (atomic_read(&rdev->irq.pflip[1]))
6667 radeon_crtc_handle_flip(rdev, 1);
6668 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6669 DRM_DEBUG("IH: D2 vblank\n");
6670 }
6671 break;
6672 case 1: /* D2 vline */
6673 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6674 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6675 DRM_DEBUG("IH: D2 vline\n");
6676 }
6677 break;
6678 default:
6679 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6680 break;
6681 }
6682 break;
6683 case 3: /* D3 vblank/vline */
6684 switch (src_data) {
6685 case 0: /* D3 vblank */
6686 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6687 if (rdev->irq.crtc_vblank_int[2]) {
6688 drm_handle_vblank(rdev->ddev, 2);
6689 rdev->pm.vblank_sync = true;
6690 wake_up(&rdev->irq.vblank_queue);
6691 }
6692 if (atomic_read(&rdev->irq.pflip[2]))
6693 radeon_crtc_handle_flip(rdev, 2);
6694 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6695 DRM_DEBUG("IH: D3 vblank\n");
6696 }
6697 break;
6698 case 1: /* D3 vline */
6699 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6700 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6701 DRM_DEBUG("IH: D3 vline\n");
6702 }
6703 break;
6704 default:
6705 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6706 break;
6707 }
6708 break;
6709 case 4: /* D4 vblank/vline */
6710 switch (src_data) {
6711 case 0: /* D4 vblank */
6712 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6713 if (rdev->irq.crtc_vblank_int[3]) {
6714 drm_handle_vblank(rdev->ddev, 3);
6715 rdev->pm.vblank_sync = true;
6716 wake_up(&rdev->irq.vblank_queue);
6717 }
6718 if (atomic_read(&rdev->irq.pflip[3]))
6719 radeon_crtc_handle_flip(rdev, 3);
6720 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6721 DRM_DEBUG("IH: D4 vblank\n");
6722 }
6723 break;
6724 case 1: /* D4 vline */
6725 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6726 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6727 DRM_DEBUG("IH: D4 vline\n");
6728 }
6729 break;
6730 default:
6731 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6732 break;
6733 }
6734 break;
6735 case 5: /* D5 vblank/vline */
6736 switch (src_data) {
6737 case 0: /* D5 vblank */
6738 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6739 if (rdev->irq.crtc_vblank_int[4]) {
6740 drm_handle_vblank(rdev->ddev, 4);
6741 rdev->pm.vblank_sync = true;
6742 wake_up(&rdev->irq.vblank_queue);
6743 }
6744 if (atomic_read(&rdev->irq.pflip[4]))
6745 radeon_crtc_handle_flip(rdev, 4);
6746 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6747 DRM_DEBUG("IH: D5 vblank\n");
6748 }
6749 break;
6750 case 1: /* D5 vline */
6751 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6752 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6753 DRM_DEBUG("IH: D5 vline\n");
6754 }
6755 break;
6756 default:
6757 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6758 break;
6759 }
6760 break;
6761 case 6: /* D6 vblank/vline */
6762 switch (src_data) {
6763 case 0: /* D6 vblank */
6764 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6765 if (rdev->irq.crtc_vblank_int[5]) {
6766 drm_handle_vblank(rdev->ddev, 5);
6767 rdev->pm.vblank_sync = true;
6768 wake_up(&rdev->irq.vblank_queue);
6769 }
6770 if (atomic_read(&rdev->irq.pflip[5]))
6771 radeon_crtc_handle_flip(rdev, 5);
6772 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6773 DRM_DEBUG("IH: D6 vblank\n");
6774 }
6775 break;
6776 case 1: /* D6 vline */
6777 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6778 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6779 DRM_DEBUG("IH: D6 vline\n");
6780 }
6781 break;
6782 default:
6783 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6784 break;
6785 }
6786 break;
6787 case 42: /* HPD hotplug */
6788 switch (src_data) {
6789 case 0:
6790 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6791 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
6792 queue_hotplug = true;
6793 DRM_DEBUG("IH: HPD1\n");
6794 }
6795 break;
6796 case 1:
6797 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6798 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6799 queue_hotplug = true;
6800 DRM_DEBUG("IH: HPD2\n");
6801 }
6802 break;
6803 case 2:
6804 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6805 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6806 queue_hotplug = true;
6807 DRM_DEBUG("IH: HPD3\n");
6808 }
6809 break;
6810 case 3:
6811 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6812 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6813 queue_hotplug = true;
6814 DRM_DEBUG("IH: HPD4\n");
6815 }
6816 break;
6817 case 4:
6818 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6819 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6820 queue_hotplug = true;
6821 DRM_DEBUG("IH: HPD5\n");
6822 }
6823 break;
6824 case 5:
6825 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6826 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6827 queue_hotplug = true;
6828 DRM_DEBUG("IH: HPD6\n");
6829 }
6830 break;
6831 default:
6832 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6833 break;
6834 }
6835 break;
6a3808b8
CK		 6836 case 124: /* UVD */
6837 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6838 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6839 break;
9d97c99b
AD		 6840 case 146:
6841 case 147:
3ec7d11b
AD		 6842 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6843 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6844 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9d97c99b
AD		 6845 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6846 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 6847 addr);
9d97c99b 6848 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 6849 status);
6850 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b
AD		 6851 /* reset addr and status */
6852 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6853 break;
a59781bb
AD		 6854 case 176: /* GFX RB CP_INT */
6855 case 177: /* GFX IB CP_INT */
6856 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6857 break;
6858 case 181: /* CP EOP event */
6859 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 6860 /* XXX check the bitfield order! */
6861 me_id = (ring_id & 0x60) >> 5;
6862 pipe_id = (ring_id & 0x18) >> 3;
6863 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 6864 switch (me_id) {
6865 case 0:
6866 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6867 break;
6868 case 1:
a59781bb 6869 case 2:
2b0781a6
AD		 6870 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
6871 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6872 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
6873 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 6874 break;
6875 }
6876 break;
6877 case 184: /* CP Privileged reg access */
6878 DRM_ERROR("Illegal register access in command stream\n");
6879 /* XXX check the bitfield order! */
6880 me_id = (ring_id & 0x60) >> 5;
6881 pipe_id = (ring_id & 0x18) >> 3;
6882 queue_id = (ring_id & 0x7) >> 0;
6883 switch (me_id) {
6884 case 0:
6885 /* This results in a full GPU reset, but all we need to do is soft
6886 * reset the CP for gfx
6887 */
6888 queue_reset = true;
6889 break;
6890 case 1:
6891 /* XXX compute */
2b0781a6 6892 queue_reset = true;
a59781bb
AD		 6893 break;
6894 case 2:
6895 /* XXX compute */
2b0781a6 6896 queue_reset = true;
a59781bb
AD		 6897 break;
6898 }
6899 break;
6900 case 185: /* CP Privileged inst */
6901 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 6902 /* XXX check the bitfield order! */
6903 me_id = (ring_id & 0x60) >> 5;
6904 pipe_id = (ring_id & 0x18) >> 3;
6905 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 6906 switch (me_id) {
6907 case 0:
6908 /* This results in a full GPU reset, but all we need to do is soft
6909 * reset the CP for gfx
6910 */
6911 queue_reset = true;
6912 break;
6913 case 1:
6914 /* XXX compute */
2b0781a6 6915 queue_reset = true;
a59781bb
AD		 6916 break;
6917 case 2:
6918 /* XXX compute */
2b0781a6 6919 queue_reset = true;
a59781bb
AD		 6920 break;
6921 }
6922 break;
21a93e13
AD		 6923 case 224: /* SDMA trap event */
6924 /* XXX check the bitfield order! */
6925 me_id = (ring_id & 0x3) >> 0;
6926 queue_id = (ring_id & 0xc) >> 2;
6927 DRM_DEBUG("IH: SDMA trap\n");
6928 switch (me_id) {
6929 case 0:
6930 switch (queue_id) {
6931 case 0:
6932 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6933 break;
6934 case 1:
6935 /* XXX compute */
6936 break;
6937 case 2:
6938 /* XXX compute */
6939 break;
6940 }
6941 break;
6942 case 1:
6943 switch (queue_id) {
6944 case 0:
6945 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6946 break;
6947 case 1:
6948 /* XXX compute */
6949 break;
6950 case 2:
6951 /* XXX compute */
6952 break;
6953 }
6954 break;
6955 }
6956 break;
41a524ab
AD		 6957 case 230: /* thermal low to high */
6958 DRM_DEBUG("IH: thermal low to high\n");
6959 rdev->pm.dpm.thermal.high_to_low = false;
6960 queue_thermal = true;
6961 break;
6962 case 231: /* thermal high to low */
6963 DRM_DEBUG("IH: thermal high to low\n");
6964 rdev->pm.dpm.thermal.high_to_low = true;
6965 queue_thermal = true;
6966 break;
6967 case 233: /* GUI IDLE */
6968 DRM_DEBUG("IH: GUI idle\n");
6969 break;
21a93e13
AD		 6970 case 241: /* SDMA Privileged inst */
6971 case 247: /* SDMA Privileged inst */
6972 DRM_ERROR("Illegal instruction in SDMA command stream\n");
6973 /* XXX check the bitfield order! */
6974 me_id = (ring_id & 0x3) >> 0;
6975 queue_id = (ring_id & 0xc) >> 2;
6976 switch (me_id) {
6977 case 0:
6978 switch (queue_id) {
6979 case 0:
6980 queue_reset = true;
6981 break;
6982 case 1:
6983 /* XXX compute */
6984 queue_reset = true;
6985 break;
6986 case 2:
6987 /* XXX compute */
6988 queue_reset = true;
6989 break;
6990 }
6991 break;
6992 case 1:
6993 switch (queue_id) {
6994 case 0:
6995 queue_reset = true;
6996 break;
6997 case 1:
6998 /* XXX compute */
6999 queue_reset = true;
7000 break;
7001 case 2:
7002 /* XXX compute */
7003 queue_reset = true;
7004 break;
7005 }
7006 break;
7007 }
7008 break;
a59781bb
AD		 7009 default:
7010 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7011 break;
7012 }
7013
7014 /* wptr/rptr are in bytes! */
7015 rptr += 16;
7016 rptr &= rdev->ih.ptr_mask;
7017 }
7018 if (queue_hotplug)
7019 schedule_work(&rdev->hotplug_work);
7020 if (queue_reset)
7021 schedule_work(&rdev->reset_work);
41a524ab
AD		 7022 if (queue_thermal)
7023 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb
AD		 7024 rdev->ih.rptr = rptr;
7025 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7026 atomic_set(&rdev->ih.lock, 0);
7027
7028 /* make sure wptr hasn't changed while processing */
7029 wptr = cik_get_ih_wptr(rdev);
7030 if (wptr != rptr)
7031 goto restart_ih;
7032
7033 return IRQ_HANDLED;
7034}
7bf94a2c
AD		 7035
7036/*
7037 * startup/shutdown callbacks
7038 */
7039/**
7040 * cik_startup - program the asic to a functional state
7041 *
7042 * @rdev: radeon_device pointer
7043 *
7044 * Programs the asic to a functional state (CIK).
7045 * Called by cik_init() and cik_resume().
7046 * Returns 0 for success, error for failure.
7047 */
7048static int cik_startup(struct radeon_device *rdev)
7049{
7050 struct radeon_ring *ring;
7051 int r;
7052
8a7cd276
AD		 7053 /* enable pcie gen2/3 link */
7054 cik_pcie_gen3_enable(rdev);
7235711a
AD		 7055 /* enable aspm */
7056 cik_program_aspm(rdev);
8a7cd276 7057
e5903d39
AD		 7058 /* scratch needs to be initialized before MC */
7059 r = r600_vram_scratch_init(rdev);
7060 if (r)
7061 return r;
7062
6fab3feb
AD		 7063 cik_mc_program(rdev);
7064
7bf94a2c
AD		 7065 if (rdev->flags & RADEON_IS_IGP) {
7066 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7067 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
7068 r = cik_init_microcode(rdev);
7069 if (r) {
7070 DRM_ERROR("Failed to load firmware!\n");
7071 return r;
7072 }
7073 }
7074 } else {
7075 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7076 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7077 !rdev->mc_fw) {
7078 r = cik_init_microcode(rdev);
7079 if (r) {
7080 DRM_ERROR("Failed to load firmware!\n");
7081 return r;
7082 }
7083 }
7084
7085 r = ci_mc_load_microcode(rdev);
7086 if (r) {
7087 DRM_ERROR("Failed to load MC firmware!\n");
7088 return r;
7089 }
7090 }
7091
7bf94a2c
AD		 7092 r = cik_pcie_gart_enable(rdev);
7093 if (r)
7094 return r;
7095 cik_gpu_init(rdev);
7096
7097 /* allocate rlc buffers */
22c775ce
AD		 7098 if (rdev->flags & RADEON_IS_IGP) {
7099 if (rdev->family == CHIP_KAVERI) {
7100 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7101 rdev->rlc.reg_list_size =
7102 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7103 } else {
7104 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7105 rdev->rlc.reg_list_size =
7106 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7107 }
7108 }
7109 rdev->rlc.cs_data = ci_cs_data;
7110 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 7111 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 7112 if (r) {
7113 DRM_ERROR("Failed to init rlc BOs!\n");
7114 return r;
7115 }
7116
7117 /* allocate wb buffer */
7118 r = radeon_wb_init(rdev);
7119 if (r)
7120 return r;
7121
963e81f9
AD		 7122 /* allocate mec buffers */
7123 r = cik_mec_init(rdev);
7124 if (r) {
7125 DRM_ERROR("Failed to init MEC BOs!\n");
7126 return r;
7127 }
7128
7bf94a2c
AD		 7129 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7130 if (r) {
7131 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7132 return r;
7133 }
7134
963e81f9
AD		 7135 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7136 if (r) {
7137 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7138 return r;
7139 }
7140
7141 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7142 if (r) {
7143 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7144 return r;
7145 }
7146
7bf94a2c
AD		 7147 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7148 if (r) {
7149 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7150 return r;
7151 }
7152
7153 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7154 if (r) {
7155 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7156 return r;
7157 }
7158
2ce529da 7159 r = radeon_uvd_resume(rdev);
87167bb1 7160 if (!r) {
2ce529da
AD		 7161 r = uvd_v4_2_resume(rdev);
7162 if (!r) {
7163 r = radeon_fence_driver_start_ring(rdev,
7164 R600_RING_TYPE_UVD_INDEX);
7165 if (r)
7166 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7167 }
87167bb1
CK		 7168 }
7169 if (r)
7170 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7171
7bf94a2c
AD		 7172 /* Enable IRQ */
7173 if (!rdev->irq.installed) {
7174 r = radeon_irq_kms_init(rdev);
7175 if (r)
7176 return r;
7177 }
7178
7179 r = cik_irq_init(rdev);
7180 if (r) {
7181 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7182 radeon_irq_kms_fini(rdev);
7183 return r;
7184 }
7185 cik_irq_set(rdev);
7186
7187 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7188 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7189 CP_RB0_RPTR, CP_RB0_WPTR,
2e1e6dad 7190 RADEON_CP_PACKET2);
7bf94a2c
AD		 7191 if (r)
7192 return r;
7193
963e81f9 7194 /* set up the compute queues */
2615b53a 7195 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7196 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7197 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7198 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
2e1e6dad 7199 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7200 if (r)
7201 return r;
7202 ring->me = 1; /* first MEC */
7203 ring->pipe = 0; /* first pipe */
7204 ring->queue = 0; /* first queue */
7205 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7206
2615b53a 7207 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7208 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7209 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7210 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
2e1e6dad 7211 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7212 if (r)
7213 return r;
7214 /* dGPU only have 1 MEC */
7215 ring->me = 1; /* first MEC */
7216 ring->pipe = 0; /* first pipe */
7217 ring->queue = 1; /* second queue */
7218 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7219
7bf94a2c
AD		 7220 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7221 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7222 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7223 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
2e1e6dad 7224 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7225 if (r)
7226 return r;
7227
7228 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7229 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7230 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7231 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
2e1e6dad 7232 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7233 if (r)
7234 return r;
7235
7236 r = cik_cp_resume(rdev);
7237 if (r)
7238 return r;
7239
7240 r = cik_sdma_resume(rdev);
7241 if (r)
7242 return r;
7243
87167bb1
CK		 7244 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7245 if (ring->ring_size) {
02c9f7fa 7246 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
87167bb1 7247 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2e1e6dad 7248 RADEON_CP_PACKET2);
87167bb1 7249 if (!r)
e409b128 7250 r = uvd_v1_0_init(rdev);
87167bb1
CK		 7251 if (r)
7252 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7253 }
7254
7bf94a2c
AD		 7255 r = radeon_ib_pool_init(rdev);
7256 if (r) {
7257 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7258 return r;
7259 }
7260
7261 r = radeon_vm_manager_init(rdev);
7262 if (r) {
7263 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7264 return r;
7265 }
7266
b530602f
AD		 7267 r = dce6_audio_init(rdev);
7268 if (r)
7269 return r;
7270
7bf94a2c
AD		 7271 return 0;
7272}
7273
7274/**
7275 * cik_resume - resume the asic to a functional state
7276 *
7277 * @rdev: radeon_device pointer
7278 *
7279 * Programs the asic to a functional state (CIK).
7280 * Called at resume.
7281 * Returns 0 for success, error for failure.
7282 */
7283int cik_resume(struct radeon_device *rdev)
7284{
7285 int r;
7286
7287 /* post card */
7288 atom_asic_init(rdev->mode_info.atom_context);
7289
0aafd313
AD		 7290 /* init golden registers */
7291 cik_init_golden_registers(rdev);
7292
7bf94a2c
AD		 7293 rdev->accel_working = true;
7294 r = cik_startup(rdev);
7295 if (r) {
7296 DRM_ERROR("cik startup failed on resume\n");
7297 rdev->accel_working = false;
7298 return r;
7299 }
7300
7301 return r;
7302
7303}
7304
7305/**
7306 * cik_suspend - suspend the asic
7307 *
7308 * @rdev: radeon_device pointer
7309 *
7310 * Bring the chip into a state suitable for suspend (CIK).
7311 * Called at suspend.
7312 * Returns 0 for success.
7313 */
7314int cik_suspend(struct radeon_device *rdev)
7315{
b530602f 7316 dce6_audio_fini(rdev);
7bf94a2c
AD		 7317 radeon_vm_manager_fini(rdev);
7318 cik_cp_enable(rdev, false);
7319 cik_sdma_enable(rdev, false);
e409b128 7320 uvd_v1_0_fini(rdev);
87167bb1 7321 radeon_uvd_suspend(rdev);
473359bc
AD		 7322 cik_fini_pg(rdev);
7323 cik_fini_cg(rdev);
7bf94a2c
AD		 7324 cik_irq_suspend(rdev);
7325 radeon_wb_disable(rdev);
7326 cik_pcie_gart_disable(rdev);
7327 return 0;
7328}
7329
7330/* Plan is to move initialization in that function and use
7331 * helper function so that radeon_device_init pretty much
7332 * do nothing more than calling asic specific function. This
7333 * should also allow to remove a bunch of callback function
7334 * like vram_info.
7335 */
7336/**
7337 * cik_init - asic specific driver and hw init
7338 *
7339 * @rdev: radeon_device pointer
7340 *
7341 * Setup asic specific driver variables and program the hw
7342 * to a functional state (CIK).
7343 * Called at driver startup.
7344 * Returns 0 for success, errors for failure.
7345 */
7346int cik_init(struct radeon_device *rdev)
7347{
7348 struct radeon_ring *ring;
7349 int r;
7350
7351 /* Read BIOS */
7352 if (!radeon_get_bios(rdev)) {
7353 if (ASIC_IS_AVIVO(rdev))
7354 return -EINVAL;
7355 }
7356 /* Must be an ATOMBIOS */
7357 if (!rdev->is_atom_bios) {
7358 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7359 return -EINVAL;
7360 }
7361 r = radeon_atombios_init(rdev);
7362 if (r)
7363 return r;
7364
7365 /* Post card if necessary */
7366 if (!radeon_card_posted(rdev)) {
7367 if (!rdev->bios) {
7368 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7369 return -EINVAL;
7370 }
7371 DRM_INFO("GPU not posted. posting now...\n");
7372 atom_asic_init(rdev->mode_info.atom_context);
7373 }
0aafd313
AD		 7374 /* init golden registers */
7375 cik_init_golden_registers(rdev);
7bf94a2c
AD		 7376 /* Initialize scratch registers */
7377 cik_scratch_init(rdev);
7378 /* Initialize surface registers */
7379 radeon_surface_init(rdev);
7380 /* Initialize clocks */
7381 radeon_get_clock_info(rdev->ddev);
7382
7383 /* Fence driver */
7384 r = radeon_fence_driver_init(rdev);
7385 if (r)
7386 return r;
7387
7388 /* initialize memory controller */
7389 r = cik_mc_init(rdev);
7390 if (r)
7391 return r;
7392 /* Memory manager */
7393 r = radeon_bo_init(rdev);
7394 if (r)
7395 return r;
7396
7397 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7398 ring->ring_obj = NULL;
7399 r600_ring_init(rdev, ring, 1024 * 1024);
7400
963e81f9
AD		 7401 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7402 ring->ring_obj = NULL;
7403 r600_ring_init(rdev, ring, 1024 * 1024);
7404 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7405 if (r)
7406 return r;
7407
7408 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7409 ring->ring_obj = NULL;
7410 r600_ring_init(rdev, ring, 1024 * 1024);
7411 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7412 if (r)
7413 return r;
7414
7bf94a2c
AD		 7415 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7416 ring->ring_obj = NULL;
7417 r600_ring_init(rdev, ring, 256 * 1024);
7418
7419 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7420 ring->ring_obj = NULL;
7421 r600_ring_init(rdev, ring, 256 * 1024);
7422
87167bb1
CK		 7423 r = radeon_uvd_init(rdev);
7424 if (!r) {
7425 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7426 ring->ring_obj = NULL;
7427 r600_ring_init(rdev, ring, 4096);
7428 }
7429
7bf94a2c
AD		 7430 rdev->ih.ring_obj = NULL;
7431 r600_ih_ring_init(rdev, 64 * 1024);
7432
7433 r = r600_pcie_gart_init(rdev);
7434 if (r)
7435 return r;
7436
7437 rdev->accel_working = true;
7438 r = cik_startup(rdev);
7439 if (r) {
7440 dev_err(rdev->dev, "disabling GPU acceleration\n");
7441 cik_cp_fini(rdev);
7442 cik_sdma_fini(rdev);
7443 cik_irq_fini(rdev);
1fd11777 7444 sumo_rlc_fini(rdev);
963e81f9 7445 cik_mec_fini(rdev);
7bf94a2c
AD		 7446 radeon_wb_fini(rdev);
7447 radeon_ib_pool_fini(rdev);
7448 radeon_vm_manager_fini(rdev);
7449 radeon_irq_kms_fini(rdev);
7450 cik_pcie_gart_fini(rdev);
7451 rdev->accel_working = false;
7452 }
7453
7454 /* Don't start up if the MC ucode is missing.
7455 * The default clocks and voltages before the MC ucode
7456 * is loaded are not suffient for advanced operations.
7457 */
7458 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7459 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7460 return -EINVAL;
7461 }
7462
7463 return 0;
7464}
7465
7466/**
7467 * cik_fini - asic specific driver and hw fini
7468 *
7469 * @rdev: radeon_device pointer
7470 *
7471 * Tear down the asic specific driver variables and program the hw
7472 * to an idle state (CIK).
7473 * Called at driver unload.
7474 */
7475void cik_fini(struct radeon_device *rdev)
7476{
7477 cik_cp_fini(rdev);
7478 cik_sdma_fini(rdev);
473359bc
AD		 7479 cik_fini_pg(rdev);
7480 cik_fini_cg(rdev);
7bf94a2c 7481 cik_irq_fini(rdev);
1fd11777 7482 sumo_rlc_fini(rdev);
963e81f9 7483 cik_mec_fini(rdev);
7bf94a2c
AD		 7484 radeon_wb_fini(rdev);
7485 radeon_vm_manager_fini(rdev);
7486 radeon_ib_pool_fini(rdev);
7487 radeon_irq_kms_fini(rdev);
e409b128 7488 uvd_v1_0_fini(rdev);
87167bb1 7489 radeon_uvd_fini(rdev);
7bf94a2c
AD		 7490 cik_pcie_gart_fini(rdev);
7491 r600_vram_scratch_fini(rdev);
7492 radeon_gem_fini(rdev);
7493 radeon_fence_driver_fini(rdev);
7494 radeon_bo_fini(rdev);
7495 radeon_atombios_fini(rdev);
7496 kfree(rdev->bios);
7497 rdev->bios = NULL;
7498}
cd84a27d 7499
134b480f
AD		 7500void dce8_program_fmt(struct drm_encoder *encoder)
7501{
7502 struct drm_device *dev = encoder->dev;
7503 struct radeon_device *rdev = dev->dev_private;
7504 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
7505 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
7506 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
7507 int bpc = 0;
7508 u32 tmp = 0;
6214bb74 7509 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 7510
6214bb74
AD		 7511 if (connector) {
7512 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 7513 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 7514 dither = radeon_connector->dither;
7515 }
134b480f
AD		 7516
7517 /* LVDS/eDP FMT is set up by atom */
7518 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
7519 return;
7520
7521 /* not needed for analog */
7522 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
7523 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
7524 return;
7525
7526 if (bpc == 0)
7527 return;
7528
7529 switch (bpc) {
7530 case 6:
6214bb74 7531 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7532 /* XXX sort out optimal dither settings */
7533 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7534 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
7535 else
7536 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
7537 break;
7538 case 8:
6214bb74 7539 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7540 /* XXX sort out optimal dither settings */
7541 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7542 FMT_RGB_RANDOM_ENABLE |
7543 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
7544 else
7545 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
7546 break;
7547 case 10:
6214bb74 7548 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7549 /* XXX sort out optimal dither settings */
7550 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7551 FMT_RGB_RANDOM_ENABLE |
7552 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
7553 else
7554 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
7555 break;
7556 default:
7557 /* not needed */
7558 break;
7559 }
7560
7561 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
7562}
7563
cd84a27d
AD		 7564/* display watermark setup */
7565/**
7566 * dce8_line_buffer_adjust - Set up the line buffer
7567 *
7568 * @rdev: radeon_device pointer
7569 * @radeon_crtc: the selected display controller
7570 * @mode: the current display mode on the selected display
7571 * controller
7572 *
7573 * Setup up the line buffer allocation for
7574 * the selected display controller (CIK).
7575 * Returns the line buffer size in pixels.
7576 */
7577static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
7578 struct radeon_crtc *radeon_crtc,
7579 struct drm_display_mode *mode)
7580{
bc01a8c7
AD		 7581 u32 tmp, buffer_alloc, i;
7582 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 7583 /*
7584 * Line Buffer Setup
7585 * There are 6 line buffers, one for each display controllers.
7586 * There are 3 partitions per LB. Select the number of partitions
7587 * to enable based on the display width. For display widths larger
7588 * than 4096, you need use to use 2 display controllers and combine
7589 * them using the stereo blender.
7590 */
7591 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 7592 if (mode->crtc_hdisplay < 1920) {
cd84a27d 7593 tmp = 1;
bc01a8c7
AD		 7594 buffer_alloc = 2;
7595 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 7596 tmp = 2;
bc01a8c7
AD		 7597 buffer_alloc = 2;
7598 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 7599 tmp = 0;
bc01a8c7
AD		 7600 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
7601 } else {
cd84a27d
AD		 7602 DRM_DEBUG_KMS("Mode too big for LB!\n");
7603 tmp = 0;
bc01a8c7 7604 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 7605 }
bc01a8c7 7606 } else {
cd84a27d 7607 tmp = 1;
bc01a8c7
AD		 7608 buffer_alloc = 0;
7609 }
cd84a27d
AD		 7610
7611 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
7612 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
7613
bc01a8c7
AD		 7614 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
7615 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
7616 for (i = 0; i < rdev->usec_timeout; i++) {
7617 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
7618 DMIF_BUFFERS_ALLOCATED_COMPLETED)
7619 break;
7620 udelay(1);
7621 }
7622
cd84a27d
AD		 7623 if (radeon_crtc->base.enabled && mode) {
7624 switch (tmp) {
7625 case 0:
7626 default:
7627 return 4096 * 2;
7628 case 1:
7629 return 1920 * 2;
7630 case 2:
7631 return 2560 * 2;
7632 }
7633 }
7634
7635 /* controller not enabled, so no lb used */
7636 return 0;
7637}
7638
7639/**
7640 * cik_get_number_of_dram_channels - get the number of dram channels
7641 *
7642 * @rdev: radeon_device pointer
7643 *
7644 * Look up the number of video ram channels (CIK).
7645 * Used for display watermark bandwidth calculations
7646 * Returns the number of dram channels
7647 */
7648static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
7649{
7650 u32 tmp = RREG32(MC_SHARED_CHMAP);
7651
7652 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
7653 case 0:
7654 default:
7655 return 1;
7656 case 1:
7657 return 2;
7658 case 2:
7659 return 4;
7660 case 3:
7661 return 8;
7662 case 4:
7663 return 3;
7664 case 5:
7665 return 6;
7666 case 6:
7667 return 10;
7668 case 7:
7669 return 12;
7670 case 8:
7671 return 16;
7672 }
7673}
7674
7675struct dce8_wm_params {
7676 u32 dram_channels; /* number of dram channels */
7677 u32 yclk; /* bandwidth per dram data pin in kHz */
7678 u32 sclk; /* engine clock in kHz */
7679 u32 disp_clk; /* display clock in kHz */
7680 u32 src_width; /* viewport width */
7681 u32 active_time; /* active display time in ns */
7682 u32 blank_time; /* blank time in ns */
7683 bool interlaced; /* mode is interlaced */
7684 fixed20_12 vsc; /* vertical scale ratio */
7685 u32 num_heads; /* number of active crtcs */
7686 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
7687 u32 lb_size; /* line buffer allocated to pipe */
7688 u32 vtaps; /* vertical scaler taps */
7689};
7690
7691/**
7692 * dce8_dram_bandwidth - get the dram bandwidth
7693 *
7694 * @wm: watermark calculation data
7695 *
7696 * Calculate the raw dram bandwidth (CIK).
7697 * Used for display watermark bandwidth calculations
7698 * Returns the dram bandwidth in MBytes/s
7699 */
7700static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
7701{
7702 /* Calculate raw DRAM Bandwidth */
7703 fixed20_12 dram_efficiency; /* 0.7 */
7704 fixed20_12 yclk, dram_channels, bandwidth;
7705 fixed20_12 a;
7706
7707 a.full = dfixed_const(1000);
7708 yclk.full = dfixed_const(wm->yclk);
7709 yclk.full = dfixed_div(yclk, a);
7710 dram_channels.full = dfixed_const(wm->dram_channels * 4);
7711 a.full = dfixed_const(10);
7712 dram_efficiency.full = dfixed_const(7);
7713 dram_efficiency.full = dfixed_div(dram_efficiency, a);
7714 bandwidth.full = dfixed_mul(dram_channels, yclk);
7715 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
7716
7717 return dfixed_trunc(bandwidth);
7718}
7719
7720/**
7721 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
7722 *
7723 * @wm: watermark calculation data
7724 *
7725 * Calculate the dram bandwidth used for display (CIK).
7726 * Used for display watermark bandwidth calculations
7727 * Returns the dram bandwidth for display in MBytes/s
7728 */
7729static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7730{
7731 /* Calculate DRAM Bandwidth and the part allocated to display. */
7732 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
7733 fixed20_12 yclk, dram_channels, bandwidth;
7734 fixed20_12 a;
7735
7736 a.full = dfixed_const(1000);
7737 yclk.full = dfixed_const(wm->yclk);
7738 yclk.full = dfixed_div(yclk, a);
7739 dram_channels.full = dfixed_const(wm->dram_channels * 4);
7740 a.full = dfixed_const(10);
7741 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
7742 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
7743 bandwidth.full = dfixed_mul(dram_channels, yclk);
7744 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
7745
7746 return dfixed_trunc(bandwidth);
7747}
7748
7749/**
7750 * dce8_data_return_bandwidth - get the data return bandwidth
7751 *
7752 * @wm: watermark calculation data
7753 *
7754 * Calculate the data return bandwidth used for display (CIK).
7755 * Used for display watermark bandwidth calculations
7756 * Returns the data return bandwidth in MBytes/s
7757 */
7758static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
7759{
7760 /* Calculate the display Data return Bandwidth */
7761 fixed20_12 return_efficiency; /* 0.8 */
7762 fixed20_12 sclk, bandwidth;
7763 fixed20_12 a;
7764
7765 a.full = dfixed_const(1000);
7766 sclk.full = dfixed_const(wm->sclk);
7767 sclk.full = dfixed_div(sclk, a);
7768 a.full = dfixed_const(10);
7769 return_efficiency.full = dfixed_const(8);
7770 return_efficiency.full = dfixed_div(return_efficiency, a);
7771 a.full = dfixed_const(32);
7772 bandwidth.full = dfixed_mul(a, sclk);
7773 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
7774
7775 return dfixed_trunc(bandwidth);
7776}
7777
7778/**
7779 * dce8_dmif_request_bandwidth - get the dmif bandwidth
7780 *
7781 * @wm: watermark calculation data
7782 *
7783 * Calculate the dmif bandwidth used for display (CIK).
7784 * Used for display watermark bandwidth calculations
7785 * Returns the dmif bandwidth in MBytes/s
7786 */
7787static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
7788{
7789 /* Calculate the DMIF Request Bandwidth */
7790 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
7791 fixed20_12 disp_clk, bandwidth;
7792 fixed20_12 a, b;
7793
7794 a.full = dfixed_const(1000);
7795 disp_clk.full = dfixed_const(wm->disp_clk);
7796 disp_clk.full = dfixed_div(disp_clk, a);
7797 a.full = dfixed_const(32);
7798 b.full = dfixed_mul(a, disp_clk);
7799
7800 a.full = dfixed_const(10);
7801 disp_clk_request_efficiency.full = dfixed_const(8);
7802 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
7803
7804 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
7805
7806 return dfixed_trunc(bandwidth);
7807}
7808
7809/**
7810 * dce8_available_bandwidth - get the min available bandwidth
7811 *
7812 * @wm: watermark calculation data
7813 *
7814 * Calculate the min available bandwidth used for display (CIK).
7815 * Used for display watermark bandwidth calculations
7816 * Returns the min available bandwidth in MBytes/s
7817 */
7818static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
7819{
7820 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
7821 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
7822 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
7823 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
7824
7825 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
7826}
7827
7828/**
7829 * dce8_average_bandwidth - get the average available bandwidth
7830 *
7831 * @wm: watermark calculation data
7832 *
7833 * Calculate the average available bandwidth used for display (CIK).
7834 * Used for display watermark bandwidth calculations
7835 * Returns the average available bandwidth in MBytes/s
7836 */
7837static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
7838{
7839 /* Calculate the display mode Average Bandwidth
7840 * DisplayMode should contain the source and destination dimensions,
7841 * timing, etc.
7842 */
7843 fixed20_12 bpp;
7844 fixed20_12 line_time;
7845 fixed20_12 src_width;
7846 fixed20_12 bandwidth;
7847 fixed20_12 a;
7848
7849 a.full = dfixed_const(1000);
7850 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
7851 line_time.full = dfixed_div(line_time, a);
7852 bpp.full = dfixed_const(wm->bytes_per_pixel);
7853 src_width.full = dfixed_const(wm->src_width);
7854 bandwidth.full = dfixed_mul(src_width, bpp);
7855 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
7856 bandwidth.full = dfixed_div(bandwidth, line_time);
7857
7858 return dfixed_trunc(bandwidth);
7859}
7860
7861/**
7862 * dce8_latency_watermark - get the latency watermark
7863 *
7864 * @wm: watermark calculation data
7865 *
7866 * Calculate the latency watermark (CIK).
7867 * Used for display watermark bandwidth calculations
7868 * Returns the latency watermark in ns
7869 */
7870static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
7871{
7872 /* First calculate the latency in ns */
7873 u32 mc_latency = 2000; /* 2000 ns. */
7874 u32 available_bandwidth = dce8_available_bandwidth(wm);
7875 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
7876 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
7877 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
7878 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
7879 (wm->num_heads * cursor_line_pair_return_time);
7880 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
7881 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
7882 u32 tmp, dmif_size = 12288;
7883 fixed20_12 a, b, c;
7884
7885 if (wm->num_heads == 0)
7886 return 0;
7887
7888 a.full = dfixed_const(2);
7889 b.full = dfixed_const(1);
7890 if ((wm->vsc.full > a.full) ||
7891 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
7892 (wm->vtaps >= 5) ||
7893 ((wm->vsc.full >= a.full) && wm->interlaced))
7894 max_src_lines_per_dst_line = 4;
7895 else
7896 max_src_lines_per_dst_line = 2;
7897
7898 a.full = dfixed_const(available_bandwidth);
7899 b.full = dfixed_const(wm->num_heads);
7900 a.full = dfixed_div(a, b);
7901
7902 b.full = dfixed_const(mc_latency + 512);
7903 c.full = dfixed_const(wm->disp_clk);
7904 b.full = dfixed_div(b, c);
7905
7906 c.full = dfixed_const(dmif_size);
7907 b.full = dfixed_div(c, b);
7908
7909 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
7910
7911 b.full = dfixed_const(1000);
7912 c.full = dfixed_const(wm->disp_clk);
7913 b.full = dfixed_div(c, b);
7914 c.full = dfixed_const(wm->bytes_per_pixel);
7915 b.full = dfixed_mul(b, c);
7916
7917 lb_fill_bw = min(tmp, dfixed_trunc(b));
7918
7919 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
7920 b.full = dfixed_const(1000);
7921 c.full = dfixed_const(lb_fill_bw);
7922 b.full = dfixed_div(c, b);
7923 a.full = dfixed_div(a, b);
7924 line_fill_time = dfixed_trunc(a);
7925
7926 if (line_fill_time < wm->active_time)
7927 return latency;
7928 else
7929 return latency + (line_fill_time - wm->active_time);
7930
7931}
7932
7933/**
7934 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
7935 * average and available dram bandwidth
7936 *
7937 * @wm: watermark calculation data
7938 *
7939 * Check if the display average bandwidth fits in the display
7940 * dram bandwidth (CIK).
7941 * Used for display watermark bandwidth calculations
7942 * Returns true if the display fits, false if not.
7943 */
7944static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7945{
7946 if (dce8_average_bandwidth(wm) <=
7947 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
7948 return true;
7949 else
7950 return false;
7951}
7952
7953/**
7954 * dce8_average_bandwidth_vs_available_bandwidth - check
7955 * average and available bandwidth
7956 *
7957 * @wm: watermark calculation data
7958 *
7959 * Check if the display average bandwidth fits in the display
7960 * available bandwidth (CIK).
7961 * Used for display watermark bandwidth calculations
7962 * Returns true if the display fits, false if not.
7963 */
7964static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
7965{
7966 if (dce8_average_bandwidth(wm) <=
7967 (dce8_available_bandwidth(wm) / wm->num_heads))
7968 return true;
7969 else
7970 return false;
7971}
7972
7973/**
7974 * dce8_check_latency_hiding - check latency hiding
7975 *
7976 * @wm: watermark calculation data
7977 *
7978 * Check latency hiding (CIK).
7979 * Used for display watermark bandwidth calculations
7980 * Returns true if the display fits, false if not.
7981 */
7982static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
7983{
7984 u32 lb_partitions = wm->lb_size / wm->src_width;
7985 u32 line_time = wm->active_time + wm->blank_time;
7986 u32 latency_tolerant_lines;
7987 u32 latency_hiding;
7988 fixed20_12 a;
7989
7990 a.full = dfixed_const(1);
7991 if (wm->vsc.full > a.full)
7992 latency_tolerant_lines = 1;
7993 else {
7994 if (lb_partitions <= (wm->vtaps + 1))
7995 latency_tolerant_lines = 1;
7996 else
7997 latency_tolerant_lines = 2;
7998 }
7999
8000 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8001
8002 if (dce8_latency_watermark(wm) <= latency_hiding)
8003 return true;
8004 else
8005 return false;
8006}
8007
8008/**
8009 * dce8_program_watermarks - program display watermarks
8010 *
8011 * @rdev: radeon_device pointer
8012 * @radeon_crtc: the selected display controller
8013 * @lb_size: line buffer size
8014 * @num_heads: number of display controllers in use
8015 *
8016 * Calculate and program the display watermarks for the
8017 * selected display controller (CIK).
8018 */
8019static void dce8_program_watermarks(struct radeon_device *rdev,
8020 struct radeon_crtc *radeon_crtc,
8021 u32 lb_size, u32 num_heads)
8022{
8023 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 8024 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 8025 u32 pixel_period;
8026 u32 line_time = 0;
8027 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8028 u32 tmp, wm_mask;
8029
8030 if (radeon_crtc->base.enabled && num_heads && mode) {
8031 pixel_period = 1000000 / (u32)mode->clock;
8032 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8033
58ea2dea
AD		 8034 /* watermark for high clocks */
8035 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8036 rdev->pm.dpm_enabled) {
8037 wm_high.yclk =
8038 radeon_dpm_get_mclk(rdev, false) * 10;
8039 wm_high.sclk =
8040 radeon_dpm_get_sclk(rdev, false) * 10;
8041 } else {
8042 wm_high.yclk = rdev->pm.current_mclk * 10;
8043 wm_high.sclk = rdev->pm.current_sclk * 10;
8044 }
8045
8046 wm_high.disp_clk = mode->clock;
8047 wm_high.src_width = mode->crtc_hdisplay;
8048 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8049 wm_high.blank_time = line_time - wm_high.active_time;
8050 wm_high.interlaced = false;
cd84a27d 8051 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 8052 wm_high.interlaced = true;
8053 wm_high.vsc = radeon_crtc->vsc;
8054 wm_high.vtaps = 1;
cd84a27d 8055 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 8056 wm_high.vtaps = 2;
8057 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8058 wm_high.lb_size = lb_size;
8059 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8060 wm_high.num_heads = num_heads;
cd84a27d
AD		 8061
8062 /* set for high clocks */
58ea2dea
AD		 8063 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8064
8065 /* possibly force display priority to high */
8066 /* should really do this at mode validation time... */
8067 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8068 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8069 !dce8_check_latency_hiding(&wm_high) ||
8070 (rdev->disp_priority == 2)) {
8071 DRM_DEBUG_KMS("force priority to high\n");
8072 }
8073
8074 /* watermark for low clocks */
8075 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8076 rdev->pm.dpm_enabled) {
8077 wm_low.yclk =
8078 radeon_dpm_get_mclk(rdev, true) * 10;
8079 wm_low.sclk =
8080 radeon_dpm_get_sclk(rdev, true) * 10;
8081 } else {
8082 wm_low.yclk = rdev->pm.current_mclk * 10;
8083 wm_low.sclk = rdev->pm.current_sclk * 10;
8084 }
8085
8086 wm_low.disp_clk = mode->clock;
8087 wm_low.src_width = mode->crtc_hdisplay;
8088 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8089 wm_low.blank_time = line_time - wm_low.active_time;
8090 wm_low.interlaced = false;
8091 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8092 wm_low.interlaced = true;
8093 wm_low.vsc = radeon_crtc->vsc;
8094 wm_low.vtaps = 1;
8095 if (radeon_crtc->rmx_type != RMX_OFF)
8096 wm_low.vtaps = 2;
8097 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8098 wm_low.lb_size = lb_size;
8099 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8100 wm_low.num_heads = num_heads;
8101
cd84a27d 8102 /* set for low clocks */
58ea2dea 8103 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 8104
8105 /* possibly force display priority to high */
8106 /* should really do this at mode validation time... */
58ea2dea
AD		 8107 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8108 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8109 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 8110 (rdev->disp_priority == 2)) {
8111 DRM_DEBUG_KMS("force priority to high\n");
8112 }
8113 }
8114
8115 /* select wm A */
8116 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8117 tmp = wm_mask;
8118 tmp &= ~LATENCY_WATERMARK_MASK(3);
8119 tmp |= LATENCY_WATERMARK_MASK(1);
8120 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8121 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8122 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8123 LATENCY_HIGH_WATERMARK(line_time)));
8124 /* select wm B */
8125 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8126 tmp &= ~LATENCY_WATERMARK_MASK(3);
8127 tmp |= LATENCY_WATERMARK_MASK(2);
8128 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8129 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8130 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8131 LATENCY_HIGH_WATERMARK(line_time)));
8132 /* restore original selection */
8133 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 8134
8135 /* save values for DPM */
8136 radeon_crtc->line_time = line_time;
8137 radeon_crtc->wm_high = latency_watermark_a;
8138 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 8139}
8140
8141/**
8142 * dce8_bandwidth_update - program display watermarks
8143 *
8144 * @rdev: radeon_device pointer
8145 *
8146 * Calculate and program the display watermarks and line
8147 * buffer allocation (CIK).
8148 */
8149void dce8_bandwidth_update(struct radeon_device *rdev)
8150{
8151 struct drm_display_mode *mode = NULL;
8152 u32 num_heads = 0, lb_size;
8153 int i;
8154
8155 radeon_update_display_priority(rdev);
8156
8157 for (i = 0; i < rdev->num_crtc; i++) {
8158 if (rdev->mode_info.crtcs[i]->base.enabled)
8159 num_heads++;
8160 }
8161 for (i = 0; i < rdev->num_crtc; i++) {
8162 mode = &rdev->mode_info.crtcs[i]->base.mode;
8163 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8164 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8165 }
8166}
44fa346f
AD		 8167
8168/**
8169 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8170 *
8171 * @rdev: radeon_device pointer
8172 *
8173 * Fetches a GPU clock counter snapshot (SI).
8174 * Returns the 64 bit clock counter snapshot.
8175 */
8176uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8177{
8178 uint64_t clock;
8179
8180 mutex_lock(&rdev->gpu_clock_mutex);
8181 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8182 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8183 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8184 mutex_unlock(&rdev->gpu_clock_mutex);
8185 return clock;
8186}
8187
87167bb1
CK		 8188static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8189 u32 cntl_reg, u32 status_reg)
8190{
8191 int r, i;
8192 struct atom_clock_dividers dividers;
8193 uint32_t tmp;
8194
8195 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8196 clock, false, &dividers);
8197 if (r)
8198 return r;
8199
8200 tmp = RREG32_SMC(cntl_reg);
8201 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8202 tmp |= dividers.post_divider;
8203 WREG32_SMC(cntl_reg, tmp);
8204
8205 for (i = 0; i < 100; i++) {
8206 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8207 break;
8208 mdelay(10);
8209 }
8210 if (i == 100)
8211 return -ETIMEDOUT;
8212
8213 return 0;
8214}
8215
8216int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8217{
8218 int r = 0;
8219
8220 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8221 if (r)
8222 return r;
8223
8224 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8225 return r;
8226}
8227
8a7cd276 8228static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 8229{
8a7cd276
AD		 8230 struct pci_dev *root = rdev->pdev->bus->self;
8231 int bridge_pos, gpu_pos;
8232 u32 speed_cntl, mask, current_data_rate;
8233 int ret, i;
8234 u16 tmp16;
87167bb1 8235
8a7cd276
AD		 8236 if (radeon_pcie_gen2 == 0)
8237 return;
87167bb1 8238
8a7cd276
AD		 8239 if (rdev->flags & RADEON_IS_IGP)
8240 return;
87167bb1 8241
8a7cd276
AD		 8242 if (!(rdev->flags & RADEON_IS_PCIE))
8243 return;
87167bb1 8244
8a7cd276
AD		 8245 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8246 if (ret != 0)
8247 return;
87167bb1 8248
8a7cd276
AD		 8249 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8250 return;
87167bb1 8251
8a7cd276
AD		 8252 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8253 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8254 LC_CURRENT_DATA_RATE_SHIFT;
8255 if (mask & DRM_PCIE_SPEED_80) {
8256 if (current_data_rate == 2) {
8257 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8258 return;
8259 }
8260 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8261 } else if (mask & DRM_PCIE_SPEED_50) {
8262 if (current_data_rate == 1) {
8263 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8264 return;
8265 }
8266 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8267 }
87167bb1 8268
8a7cd276
AD		 8269 bridge_pos = pci_pcie_cap(root);
8270 if (!bridge_pos)
8271 return;
8272
8273 gpu_pos = pci_pcie_cap(rdev->pdev);
8274 if (!gpu_pos)
8275 return;
8276
8277 if (mask & DRM_PCIE_SPEED_80) {
8278 /* re-try equalization if gen3 is not already enabled */
8279 if (current_data_rate != 2) {
8280 u16 bridge_cfg, gpu_cfg;
8281 u16 bridge_cfg2, gpu_cfg2;
8282 u32 max_lw, current_lw, tmp;
8283
8284 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8285 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8286
8287 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8288 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8289
8290 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8291 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8292
8293 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8294 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8295 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8296
8297 if (current_lw < max_lw) {
8298 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8299 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8300 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8301 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8302 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8303 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8304 }
8305 }
8306
8307 for (i = 0; i < 10; i++) {
8308 /* check status */
8309 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8310 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8311 break;
8312
8313 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8314 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8315
8316 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8317 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8318
8319 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8320 tmp |= LC_SET_QUIESCE;
8321 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8322
8323 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8324 tmp |= LC_REDO_EQ;
8325 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8326
8327 mdelay(100);
8328
8329 /* linkctl */
8330 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8331 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8332 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8333 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8334
8335 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8336 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8337 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8338 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8339
8340 /* linkctl2 */
8341 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8342 tmp16 &= ~((1 << 4) | (7 << 9));
8343 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8344 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8345
8346 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8347 tmp16 &= ~((1 << 4) | (7 << 9));
8348 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8349 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8350
8351 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8352 tmp &= ~LC_SET_QUIESCE;
8353 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8354 }
8355 }
8356 }
8357
8358 /* set the link speed */
8359 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8360 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8361 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8362
8363 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8364 tmp16 &= ~0xf;
8365 if (mask & DRM_PCIE_SPEED_80)
8366 tmp16 |= 3; /* gen3 */
8367 else if (mask & DRM_PCIE_SPEED_50)
8368 tmp16 |= 2; /* gen2 */
8369 else
8370 tmp16 |= 1; /* gen1 */
8371 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8372
8373 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8374 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8375 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8376
8377 for (i = 0; i < rdev->usec_timeout; i++) {
8378 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8379 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8380 break;
8381 udelay(1);
8382 }
8383}
7235711a
AD		 8384
8385static void cik_program_aspm(struct radeon_device *rdev)
8386{
8387 u32 data, orig;
8388 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8389 bool disable_clkreq = false;
8390
8391 if (radeon_aspm == 0)
8392 return;
8393
8394 /* XXX double check IGPs */
8395 if (rdev->flags & RADEON_IS_IGP)
8396 return;
8397
8398 if (!(rdev->flags & RADEON_IS_PCIE))
8399 return;
8400
8401 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8402 data &= ~LC_XMIT_N_FTS_MASK;
8403 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8404 if (orig != data)
8405 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8406
8407 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8408 data |= LC_GO_TO_RECOVERY;
8409 if (orig != data)
8410 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8411
8412 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8413 data |= P_IGNORE_EDB_ERR;
8414 if (orig != data)
8415 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8416
8417 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8418 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8419 data |= LC_PMI_TO_L1_DIS;
8420 if (!disable_l0s)
8421 data |= LC_L0S_INACTIVITY(7);
8422
8423 if (!disable_l1) {
8424 data |= LC_L1_INACTIVITY(7);
8425 data &= ~LC_PMI_TO_L1_DIS;
8426 if (orig != data)
8427 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8428
8429 if (!disable_plloff_in_l1) {
8430 bool clk_req_support;
8431
8432 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8433 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8434 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8435 if (orig != data)
8436 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8437
8438 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8439 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8440 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8441 if (orig != data)
8442 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8443
8444 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8445 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8446 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8447 if (orig != data)
8448 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8449
8450 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8451 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8452 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8453 if (orig != data)
8454 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8455
8456 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8457 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8458 data |= LC_DYN_LANES_PWR_STATE(3);
8459 if (orig != data)
8460 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8461
8462 if (!disable_clkreq) {
8463 struct pci_dev *root = rdev->pdev->bus->self;
8464 u32 lnkcap;
8465
8466 clk_req_support = false;
8467 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8468 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8469 clk_req_support = true;
8470 } else {
8471 clk_req_support = false;
8472 }
8473
8474 if (clk_req_support) {
8475 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8476 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8477 if (orig != data)
8478 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8479
8480 orig = data = RREG32_SMC(THM_CLK_CNTL);
8481 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8482 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8483 if (orig != data)
8484 WREG32_SMC(THM_CLK_CNTL, data);
8485
8486 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8487 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8488 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8489 if (orig != data)
8490 WREG32_SMC(MISC_CLK_CTRL, data);
8491
8492 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8493 data &= ~BCLK_AS_XCLK;
8494 if (orig != data)
8495 WREG32_SMC(CG_CLKPIN_CNTL, data);
8496
8497 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8498 data &= ~FORCE_BIF_REFCLK_EN;
8499 if (orig != data)
8500 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8501
8502 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8503 data &= ~MPLL_CLKOUT_SEL_MASK;
8504 data |= MPLL_CLKOUT_SEL(4);
8505 if (orig != data)
8506 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8507 }
8508 }
8509 } else {
8510 if (orig != data)
8511 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8512 }
8513
8514 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8515 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8516 if (orig != data)
8517 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8518
8519 if (!disable_l0s) {
8520 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8521 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8522 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8523 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8524 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8525 data &= ~LC_L0S_INACTIVITY_MASK;
8526 if (orig != data)
8527 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8528 }
8529 }
8530 }
87167bb1 8531}
Linux 6.x block layer and io_uring tree(s)