projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: Retry DDC probing on DVI on failure if we got an HPD interrupt
[linux-2.6-block.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
5a0e3ad6 25#include <linux/slab.h>
760285e7 26#include <drm/drmP.h>
bcc1c2a1 27#include "radeon.h"
e6990375 28#include "radeon_asic.h"
bfc1f97d 29#include "radeon_audio.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
9e5acbc2
DV		 38/*
39 * Indirect registers accessor
40 */
41u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
42{
43 unsigned long flags;
44 u32 r;
45
46 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
47 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
48 r = RREG32(EVERGREEN_CG_IND_DATA);
49 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
50 return r;
51}
52
53void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
54{
55 unsigned long flags;
56
57 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
58 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
59 WREG32(EVERGREEN_CG_IND_DATA, (v));
60 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
61}
62
63u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
64{
65 unsigned long flags;
66 u32 r;
67
68 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
69 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
70 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
71 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
72 return r;
73}
74
75void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
76{
77 unsigned long flags;
78
79 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
80 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
81 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
82 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
83}
84
85u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
86{
87 unsigned long flags;
88 u32 r;
89
90 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
91 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
92 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
93 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
94 return r;
95}
96
97void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
98{
99 unsigned long flags;
100
101 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
102 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
103 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
104 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
105}
106
4a15903d
AD		 107static const u32 crtc_offsets[6] =
108{
109 EVERGREEN_CRTC0_REGISTER_OFFSET,
110 EVERGREEN_CRTC1_REGISTER_OFFSET,
111 EVERGREEN_CRTC2_REGISTER_OFFSET,
112 EVERGREEN_CRTC3_REGISTER_OFFSET,
113 EVERGREEN_CRTC4_REGISTER_OFFSET,
114 EVERGREEN_CRTC5_REGISTER_OFFSET
115};
116
2948f5e6
AD		 117#include "clearstate_evergreen.h"
118
1fd11777 119static const u32 sumo_rlc_save_restore_register_list[] =
2948f5e6
AD		 120{
121 0x98fc,
122 0x9830,
123 0x9834,
124 0x9838,
125 0x9870,
126 0x9874,
127 0x8a14,
128 0x8b24,
129 0x8bcc,
130 0x8b10,
131 0x8d00,
132 0x8d04,
133 0x8c00,
134 0x8c04,
135 0x8c08,
136 0x8c0c,
137 0x8d8c,
138 0x8c20,
139 0x8c24,
140 0x8c28,
141 0x8c18,
142 0x8c1c,
143 0x8cf0,
144 0x8e2c,
145 0x8e38,
146 0x8c30,
147 0x9508,
148 0x9688,
149 0x9608,
150 0x960c,
151 0x9610,
152 0x9614,
153 0x88c4,
154 0x88d4,
155 0xa008,
156 0x900c,
157 0x9100,
158 0x913c,
159 0x98f8,
160 0x98f4,
161 0x9b7c,
162 0x3f8c,
163 0x8950,
164 0x8954,
165 0x8a18,
166 0x8b28,
167 0x9144,
168 0x9148,
169 0x914c,
170 0x3f90,
171 0x3f94,
172 0x915c,
173 0x9160,
174 0x9178,
175 0x917c,
176 0x9180,
177 0x918c,
178 0x9190,
179 0x9194,
180 0x9198,
181 0x919c,
182 0x91a8,
183 0x91ac,
184 0x91b0,
185 0x91b4,
186 0x91b8,
187 0x91c4,
188 0x91c8,
189 0x91cc,
190 0x91d0,
191 0x91d4,
192 0x91e0,
193 0x91e4,
194 0x91ec,
195 0x91f0,
196 0x91f4,
197 0x9200,
198 0x9204,
199 0x929c,
200 0x9150,
201 0x802c,
202};
2948f5e6 203
bcc1c2a1
AD		 204static void evergreen_gpu_init(struct radeon_device *rdev);
205void evergreen_fini(struct radeon_device *rdev);
b07759bf 206void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 207void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 208extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
209 int ring, u32 cp_int_cntl);
54e2e49c
AD		 210extern void cayman_vm_decode_fault(struct radeon_device *rdev,
211 u32 status, u32 addr);
22c775ce 212void cik_init_cp_pg_table(struct radeon_device *rdev);
bcc1c2a1 213
59a82d0e
AD		 214extern u32 si_get_csb_size(struct radeon_device *rdev);
215extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
a0f38609
AD		 216extern u32 cik_get_csb_size(struct radeon_device *rdev);
217extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
b5470b03 218extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
bcc1c2a1 219
d4788db3
AD		 220static const u32 evergreen_golden_registers[] =
221{
222 0x3f90, 0xffff0000, 0xff000000,
223 0x9148, 0xffff0000, 0xff000000,
224 0x3f94, 0xffff0000, 0xff000000,
225 0x914c, 0xffff0000, 0xff000000,
226 0x9b7c, 0xffffffff, 0x00000000,
227 0x8a14, 0xffffffff, 0x00000007,
228 0x8b10, 0xffffffff, 0x00000000,
229 0x960c, 0xffffffff, 0x54763210,
230 0x88c4, 0xffffffff, 0x000000c2,
231 0x88d4, 0xffffffff, 0x00000010,
232 0x8974, 0xffffffff, 0x00000000,
233 0xc78, 0x00000080, 0x00000080,
234 0x5eb4, 0xffffffff, 0x00000002,
235 0x5e78, 0xffffffff, 0x001000f0,
236 0x6104, 0x01000300, 0x00000000,
237 0x5bc0, 0x00300000, 0x00000000,
238 0x7030, 0xffffffff, 0x00000011,
239 0x7c30, 0xffffffff, 0x00000011,
240 0x10830, 0xffffffff, 0x00000011,
241 0x11430, 0xffffffff, 0x00000011,
242 0x12030, 0xffffffff, 0x00000011,
243 0x12c30, 0xffffffff, 0x00000011,
244 0xd02c, 0xffffffff, 0x08421000,
245 0x240c, 0xffffffff, 0x00000380,
246 0x8b24, 0xffffffff, 0x00ff0fff,
247 0x28a4c, 0x06000000, 0x06000000,
248 0x10c, 0x00000001, 0x00000001,
249 0x8d00, 0xffffffff, 0x100e4848,
250 0x8d04, 0xffffffff, 0x00164745,
251 0x8c00, 0xffffffff, 0xe4000003,
252 0x8c04, 0xffffffff, 0x40600060,
253 0x8c08, 0xffffffff, 0x001c001c,
254 0x8cf0, 0xffffffff, 0x08e00620,
255 0x8c20, 0xffffffff, 0x00800080,
256 0x8c24, 0xffffffff, 0x00800080,
257 0x8c18, 0xffffffff, 0x20202078,
258 0x8c1c, 0xffffffff, 0x00001010,
259 0x28350, 0xffffffff, 0x00000000,
260 0xa008, 0xffffffff, 0x00010000,
6abafb78 261 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 262 0x9508, 0xffffffff, 0x00000002,
263 0x913c, 0x0000000f, 0x0000000a
264};
265
266static const u32 evergreen_golden_registers2[] =
267{
268 0x2f4c, 0xffffffff, 0x00000000,
269 0x54f4, 0xffffffff, 0x00000000,
270 0x54f0, 0xffffffff, 0x00000000,
271 0x5498, 0xffffffff, 0x00000000,
272 0x549c, 0xffffffff, 0x00000000,
273 0x5494, 0xffffffff, 0x00000000,
274 0x53cc, 0xffffffff, 0x00000000,
275 0x53c8, 0xffffffff, 0x00000000,
276 0x53c4, 0xffffffff, 0x00000000,
277 0x53c0, 0xffffffff, 0x00000000,
278 0x53bc, 0xffffffff, 0x00000000,
279 0x53b8, 0xffffffff, 0x00000000,
280 0x53b4, 0xffffffff, 0x00000000,
281 0x53b0, 0xffffffff, 0x00000000
282};
283
284static const u32 cypress_mgcg_init[] =
285{
286 0x802c, 0xffffffff, 0xc0000000,
287 0x5448, 0xffffffff, 0x00000100,
288 0x55e4, 0xffffffff, 0x00000100,
289 0x160c, 0xffffffff, 0x00000100,
290 0x5644, 0xffffffff, 0x00000100,
291 0xc164, 0xffffffff, 0x00000100,
292 0x8a18, 0xffffffff, 0x00000100,
293 0x897c, 0xffffffff, 0x06000100,
294 0x8b28, 0xffffffff, 0x00000100,
295 0x9144, 0xffffffff, 0x00000100,
296 0x9a60, 0xffffffff, 0x00000100,
297 0x9868, 0xffffffff, 0x00000100,
298 0x8d58, 0xffffffff, 0x00000100,
299 0x9510, 0xffffffff, 0x00000100,
300 0x949c, 0xffffffff, 0x00000100,
301 0x9654, 0xffffffff, 0x00000100,
302 0x9030, 0xffffffff, 0x00000100,
303 0x9034, 0xffffffff, 0x00000100,
304 0x9038, 0xffffffff, 0x00000100,
305 0x903c, 0xffffffff, 0x00000100,
306 0x9040, 0xffffffff, 0x00000100,
307 0xa200, 0xffffffff, 0x00000100,
308 0xa204, 0xffffffff, 0x00000100,
309 0xa208, 0xffffffff, 0x00000100,
310 0xa20c, 0xffffffff, 0x00000100,
311 0x971c, 0xffffffff, 0x00000100,
312 0x977c, 0xffffffff, 0x00000100,
313 0x3f80, 0xffffffff, 0x00000100,
314 0xa210, 0xffffffff, 0x00000100,
315 0xa214, 0xffffffff, 0x00000100,
316 0x4d8, 0xffffffff, 0x00000100,
317 0x9784, 0xffffffff, 0x00000100,
318 0x9698, 0xffffffff, 0x00000100,
319 0x4d4, 0xffffffff, 0x00000200,
320 0x30cc, 0xffffffff, 0x00000100,
321 0xd0c0, 0xffffffff, 0xff000100,
322 0x802c, 0xffffffff, 0x40000000,
323 0x915c, 0xffffffff, 0x00010000,
324 0x9160, 0xffffffff, 0x00030002,
325 0x9178, 0xffffffff, 0x00070000,
326 0x917c, 0xffffffff, 0x00030002,
327 0x9180, 0xffffffff, 0x00050004,
328 0x918c, 0xffffffff, 0x00010006,
329 0x9190, 0xffffffff, 0x00090008,
330 0x9194, 0xffffffff, 0x00070000,
331 0x9198, 0xffffffff, 0x00030002,
332 0x919c, 0xffffffff, 0x00050004,
333 0x91a8, 0xffffffff, 0x00010006,
334 0x91ac, 0xffffffff, 0x00090008,
335 0x91b0, 0xffffffff, 0x00070000,
336 0x91b4, 0xffffffff, 0x00030002,
337 0x91b8, 0xffffffff, 0x00050004,
338 0x91c4, 0xffffffff, 0x00010006,
339 0x91c8, 0xffffffff, 0x00090008,
340 0x91cc, 0xffffffff, 0x00070000,
341 0x91d0, 0xffffffff, 0x00030002,
342 0x91d4, 0xffffffff, 0x00050004,
343 0x91e0, 0xffffffff, 0x00010006,
344 0x91e4, 0xffffffff, 0x00090008,
345 0x91e8, 0xffffffff, 0x00000000,
346 0x91ec, 0xffffffff, 0x00070000,
347 0x91f0, 0xffffffff, 0x00030002,
348 0x91f4, 0xffffffff, 0x00050004,
349 0x9200, 0xffffffff, 0x00010006,
350 0x9204, 0xffffffff, 0x00090008,
351 0x9208, 0xffffffff, 0x00070000,
352 0x920c, 0xffffffff, 0x00030002,
353 0x9210, 0xffffffff, 0x00050004,
354 0x921c, 0xffffffff, 0x00010006,
355 0x9220, 0xffffffff, 0x00090008,
356 0x9224, 0xffffffff, 0x00070000,
357 0x9228, 0xffffffff, 0x00030002,
358 0x922c, 0xffffffff, 0x00050004,
359 0x9238, 0xffffffff, 0x00010006,
360 0x923c, 0xffffffff, 0x00090008,
361 0x9240, 0xffffffff, 0x00070000,
362 0x9244, 0xffffffff, 0x00030002,
363 0x9248, 0xffffffff, 0x00050004,
364 0x9254, 0xffffffff, 0x00010006,
365 0x9258, 0xffffffff, 0x00090008,
366 0x925c, 0xffffffff, 0x00070000,
367 0x9260, 0xffffffff, 0x00030002,
368 0x9264, 0xffffffff, 0x00050004,
369 0x9270, 0xffffffff, 0x00010006,
370 0x9274, 0xffffffff, 0x00090008,
371 0x9278, 0xffffffff, 0x00070000,
372 0x927c, 0xffffffff, 0x00030002,
373 0x9280, 0xffffffff, 0x00050004,
374 0x928c, 0xffffffff, 0x00010006,
375 0x9290, 0xffffffff, 0x00090008,
376 0x9294, 0xffffffff, 0x00000000,
377 0x929c, 0xffffffff, 0x00000001,
378 0x802c, 0xffffffff, 0x40010000,
379 0x915c, 0xffffffff, 0x00010000,
380 0x9160, 0xffffffff, 0x00030002,
381 0x9178, 0xffffffff, 0x00070000,
382 0x917c, 0xffffffff, 0x00030002,
383 0x9180, 0xffffffff, 0x00050004,
384 0x918c, 0xffffffff, 0x00010006,
385 0x9190, 0xffffffff, 0x00090008,
386 0x9194, 0xffffffff, 0x00070000,
387 0x9198, 0xffffffff, 0x00030002,
388 0x919c, 0xffffffff, 0x00050004,
389 0x91a8, 0xffffffff, 0x00010006,
390 0x91ac, 0xffffffff, 0x00090008,
391 0x91b0, 0xffffffff, 0x00070000,
392 0x91b4, 0xffffffff, 0x00030002,
393 0x91b8, 0xffffffff, 0x00050004,
394 0x91c4, 0xffffffff, 0x00010006,
395 0x91c8, 0xffffffff, 0x00090008,
396 0x91cc, 0xffffffff, 0x00070000,
397 0x91d0, 0xffffffff, 0x00030002,
398 0x91d4, 0xffffffff, 0x00050004,
399 0x91e0, 0xffffffff, 0x00010006,
400 0x91e4, 0xffffffff, 0x00090008,
401 0x91e8, 0xffffffff, 0x00000000,
402 0x91ec, 0xffffffff, 0x00070000,
403 0x91f0, 0xffffffff, 0x00030002,
404 0x91f4, 0xffffffff, 0x00050004,
405 0x9200, 0xffffffff, 0x00010006,
406 0x9204, 0xffffffff, 0x00090008,
407 0x9208, 0xffffffff, 0x00070000,
408 0x920c, 0xffffffff, 0x00030002,
409 0x9210, 0xffffffff, 0x00050004,
410 0x921c, 0xffffffff, 0x00010006,
411 0x9220, 0xffffffff, 0x00090008,
412 0x9224, 0xffffffff, 0x00070000,
413 0x9228, 0xffffffff, 0x00030002,
414 0x922c, 0xffffffff, 0x00050004,
415 0x9238, 0xffffffff, 0x00010006,
416 0x923c, 0xffffffff, 0x00090008,
417 0x9240, 0xffffffff, 0x00070000,
418 0x9244, 0xffffffff, 0x00030002,
419 0x9248, 0xffffffff, 0x00050004,
420 0x9254, 0xffffffff, 0x00010006,
421 0x9258, 0xffffffff, 0x00090008,
422 0x925c, 0xffffffff, 0x00070000,
423 0x9260, 0xffffffff, 0x00030002,
424 0x9264, 0xffffffff, 0x00050004,
425 0x9270, 0xffffffff, 0x00010006,
426 0x9274, 0xffffffff, 0x00090008,
427 0x9278, 0xffffffff, 0x00070000,
428 0x927c, 0xffffffff, 0x00030002,
429 0x9280, 0xffffffff, 0x00050004,
430 0x928c, 0xffffffff, 0x00010006,
431 0x9290, 0xffffffff, 0x00090008,
432 0x9294, 0xffffffff, 0x00000000,
433 0x929c, 0xffffffff, 0x00000001,
434 0x802c, 0xffffffff, 0xc0000000
435};
436
437static const u32 redwood_mgcg_init[] =
438{
439 0x802c, 0xffffffff, 0xc0000000,
440 0x5448, 0xffffffff, 0x00000100,
441 0x55e4, 0xffffffff, 0x00000100,
442 0x160c, 0xffffffff, 0x00000100,
443 0x5644, 0xffffffff, 0x00000100,
444 0xc164, 0xffffffff, 0x00000100,
445 0x8a18, 0xffffffff, 0x00000100,
446 0x897c, 0xffffffff, 0x06000100,
447 0x8b28, 0xffffffff, 0x00000100,
448 0x9144, 0xffffffff, 0x00000100,
449 0x9a60, 0xffffffff, 0x00000100,
450 0x9868, 0xffffffff, 0x00000100,
451 0x8d58, 0xffffffff, 0x00000100,
452 0x9510, 0xffffffff, 0x00000100,
453 0x949c, 0xffffffff, 0x00000100,
454 0x9654, 0xffffffff, 0x00000100,
455 0x9030, 0xffffffff, 0x00000100,
456 0x9034, 0xffffffff, 0x00000100,
457 0x9038, 0xffffffff, 0x00000100,
458 0x903c, 0xffffffff, 0x00000100,
459 0x9040, 0xffffffff, 0x00000100,
460 0xa200, 0xffffffff, 0x00000100,
461 0xa204, 0xffffffff, 0x00000100,
462 0xa208, 0xffffffff, 0x00000100,
463 0xa20c, 0xffffffff, 0x00000100,
464 0x971c, 0xffffffff, 0x00000100,
465 0x977c, 0xffffffff, 0x00000100,
466 0x3f80, 0xffffffff, 0x00000100,
467 0xa210, 0xffffffff, 0x00000100,
468 0xa214, 0xffffffff, 0x00000100,
469 0x4d8, 0xffffffff, 0x00000100,
470 0x9784, 0xffffffff, 0x00000100,
471 0x9698, 0xffffffff, 0x00000100,
472 0x4d4, 0xffffffff, 0x00000200,
473 0x30cc, 0xffffffff, 0x00000100,
474 0xd0c0, 0xffffffff, 0xff000100,
475 0x802c, 0xffffffff, 0x40000000,
476 0x915c, 0xffffffff, 0x00010000,
477 0x9160, 0xffffffff, 0x00030002,
478 0x9178, 0xffffffff, 0x00070000,
479 0x917c, 0xffffffff, 0x00030002,
480 0x9180, 0xffffffff, 0x00050004,
481 0x918c, 0xffffffff, 0x00010006,
482 0x9190, 0xffffffff, 0x00090008,
483 0x9194, 0xffffffff, 0x00070000,
484 0x9198, 0xffffffff, 0x00030002,
485 0x919c, 0xffffffff, 0x00050004,
486 0x91a8, 0xffffffff, 0x00010006,
487 0x91ac, 0xffffffff, 0x00090008,
488 0x91b0, 0xffffffff, 0x00070000,
489 0x91b4, 0xffffffff, 0x00030002,
490 0x91b8, 0xffffffff, 0x00050004,
491 0x91c4, 0xffffffff, 0x00010006,
492 0x91c8, 0xffffffff, 0x00090008,
493 0x91cc, 0xffffffff, 0x00070000,
494 0x91d0, 0xffffffff, 0x00030002,
495 0x91d4, 0xffffffff, 0x00050004,
496 0x91e0, 0xffffffff, 0x00010006,
497 0x91e4, 0xffffffff, 0x00090008,
498 0x91e8, 0xffffffff, 0x00000000,
499 0x91ec, 0xffffffff, 0x00070000,
500 0x91f0, 0xffffffff, 0x00030002,
501 0x91f4, 0xffffffff, 0x00050004,
502 0x9200, 0xffffffff, 0x00010006,
503 0x9204, 0xffffffff, 0x00090008,
504 0x9294, 0xffffffff, 0x00000000,
505 0x929c, 0xffffffff, 0x00000001,
506 0x802c, 0xffffffff, 0xc0000000
507};
508
509static const u32 cedar_golden_registers[] =
510{
511 0x3f90, 0xffff0000, 0xff000000,
512 0x9148, 0xffff0000, 0xff000000,
513 0x3f94, 0xffff0000, 0xff000000,
514 0x914c, 0xffff0000, 0xff000000,
515 0x9b7c, 0xffffffff, 0x00000000,
516 0x8a14, 0xffffffff, 0x00000007,
517 0x8b10, 0xffffffff, 0x00000000,
518 0x960c, 0xffffffff, 0x54763210,
519 0x88c4, 0xffffffff, 0x000000c2,
520 0x88d4, 0xffffffff, 0x00000000,
521 0x8974, 0xffffffff, 0x00000000,
522 0xc78, 0x00000080, 0x00000080,
523 0x5eb4, 0xffffffff, 0x00000002,
524 0x5e78, 0xffffffff, 0x001000f0,
525 0x6104, 0x01000300, 0x00000000,
526 0x5bc0, 0x00300000, 0x00000000,
527 0x7030, 0xffffffff, 0x00000011,
528 0x7c30, 0xffffffff, 0x00000011,
529 0x10830, 0xffffffff, 0x00000011,
530 0x11430, 0xffffffff, 0x00000011,
531 0xd02c, 0xffffffff, 0x08421000,
532 0x240c, 0xffffffff, 0x00000380,
533 0x8b24, 0xffffffff, 0x00ff0fff,
534 0x28a4c, 0x06000000, 0x06000000,
535 0x10c, 0x00000001, 0x00000001,
536 0x8d00, 0xffffffff, 0x100e4848,
537 0x8d04, 0xffffffff, 0x00164745,
538 0x8c00, 0xffffffff, 0xe4000003,
539 0x8c04, 0xffffffff, 0x40600060,
540 0x8c08, 0xffffffff, 0x001c001c,
541 0x8cf0, 0xffffffff, 0x08e00410,
542 0x8c20, 0xffffffff, 0x00800080,
543 0x8c24, 0xffffffff, 0x00800080,
544 0x8c18, 0xffffffff, 0x20202078,
545 0x8c1c, 0xffffffff, 0x00001010,
546 0x28350, 0xffffffff, 0x00000000,
547 0xa008, 0xffffffff, 0x00010000,
6abafb78 548 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 549 0x9508, 0xffffffff, 0x00000002
550};
551
552static const u32 cedar_mgcg_init[] =
553{
554 0x802c, 0xffffffff, 0xc0000000,
555 0x5448, 0xffffffff, 0x00000100,
556 0x55e4, 0xffffffff, 0x00000100,
557 0x160c, 0xffffffff, 0x00000100,
558 0x5644, 0xffffffff, 0x00000100,
559 0xc164, 0xffffffff, 0x00000100,
560 0x8a18, 0xffffffff, 0x00000100,
561 0x897c, 0xffffffff, 0x06000100,
562 0x8b28, 0xffffffff, 0x00000100,
563 0x9144, 0xffffffff, 0x00000100,
564 0x9a60, 0xffffffff, 0x00000100,
565 0x9868, 0xffffffff, 0x00000100,
566 0x8d58, 0xffffffff, 0x00000100,
567 0x9510, 0xffffffff, 0x00000100,
568 0x949c, 0xffffffff, 0x00000100,
569 0x9654, 0xffffffff, 0x00000100,
570 0x9030, 0xffffffff, 0x00000100,
571 0x9034, 0xffffffff, 0x00000100,
572 0x9038, 0xffffffff, 0x00000100,
573 0x903c, 0xffffffff, 0x00000100,
574 0x9040, 0xffffffff, 0x00000100,
575 0xa200, 0xffffffff, 0x00000100,
576 0xa204, 0xffffffff, 0x00000100,
577 0xa208, 0xffffffff, 0x00000100,
578 0xa20c, 0xffffffff, 0x00000100,
579 0x971c, 0xffffffff, 0x00000100,
580 0x977c, 0xffffffff, 0x00000100,
581 0x3f80, 0xffffffff, 0x00000100,
582 0xa210, 0xffffffff, 0x00000100,
583 0xa214, 0xffffffff, 0x00000100,
584 0x4d8, 0xffffffff, 0x00000100,
585 0x9784, 0xffffffff, 0x00000100,
586 0x9698, 0xffffffff, 0x00000100,
587 0x4d4, 0xffffffff, 0x00000200,
588 0x30cc, 0xffffffff, 0x00000100,
589 0xd0c0, 0xffffffff, 0xff000100,
590 0x802c, 0xffffffff, 0x40000000,
591 0x915c, 0xffffffff, 0x00010000,
592 0x9178, 0xffffffff, 0x00050000,
593 0x917c, 0xffffffff, 0x00030002,
594 0x918c, 0xffffffff, 0x00010004,
595 0x9190, 0xffffffff, 0x00070006,
596 0x9194, 0xffffffff, 0x00050000,
597 0x9198, 0xffffffff, 0x00030002,
598 0x91a8, 0xffffffff, 0x00010004,
599 0x91ac, 0xffffffff, 0x00070006,
600 0x91e8, 0xffffffff, 0x00000000,
601 0x9294, 0xffffffff, 0x00000000,
602 0x929c, 0xffffffff, 0x00000001,
603 0x802c, 0xffffffff, 0xc0000000
604};
605
606static const u32 juniper_mgcg_init[] =
607{
608 0x802c, 0xffffffff, 0xc0000000,
609 0x5448, 0xffffffff, 0x00000100,
610 0x55e4, 0xffffffff, 0x00000100,
611 0x160c, 0xffffffff, 0x00000100,
612 0x5644, 0xffffffff, 0x00000100,
613 0xc164, 0xffffffff, 0x00000100,
614 0x8a18, 0xffffffff, 0x00000100,
615 0x897c, 0xffffffff, 0x06000100,
616 0x8b28, 0xffffffff, 0x00000100,
617 0x9144, 0xffffffff, 0x00000100,
618 0x9a60, 0xffffffff, 0x00000100,
619 0x9868, 0xffffffff, 0x00000100,
620 0x8d58, 0xffffffff, 0x00000100,
621 0x9510, 0xffffffff, 0x00000100,
622 0x949c, 0xffffffff, 0x00000100,
623 0x9654, 0xffffffff, 0x00000100,
624 0x9030, 0xffffffff, 0x00000100,
625 0x9034, 0xffffffff, 0x00000100,
626 0x9038, 0xffffffff, 0x00000100,
627 0x903c, 0xffffffff, 0x00000100,
628 0x9040, 0xffffffff, 0x00000100,
629 0xa200, 0xffffffff, 0x00000100,
630 0xa204, 0xffffffff, 0x00000100,
631 0xa208, 0xffffffff, 0x00000100,
632 0xa20c, 0xffffffff, 0x00000100,
633 0x971c, 0xffffffff, 0x00000100,
634 0xd0c0, 0xffffffff, 0xff000100,
635 0x802c, 0xffffffff, 0x40000000,
636 0x915c, 0xffffffff, 0x00010000,
637 0x9160, 0xffffffff, 0x00030002,
638 0x9178, 0xffffffff, 0x00070000,
639 0x917c, 0xffffffff, 0x00030002,
640 0x9180, 0xffffffff, 0x00050004,
641 0x918c, 0xffffffff, 0x00010006,
642 0x9190, 0xffffffff, 0x00090008,
643 0x9194, 0xffffffff, 0x00070000,
644 0x9198, 0xffffffff, 0x00030002,
645 0x919c, 0xffffffff, 0x00050004,
646 0x91a8, 0xffffffff, 0x00010006,
647 0x91ac, 0xffffffff, 0x00090008,
648 0x91b0, 0xffffffff, 0x00070000,
649 0x91b4, 0xffffffff, 0x00030002,
650 0x91b8, 0xffffffff, 0x00050004,
651 0x91c4, 0xffffffff, 0x00010006,
652 0x91c8, 0xffffffff, 0x00090008,
653 0x91cc, 0xffffffff, 0x00070000,
654 0x91d0, 0xffffffff, 0x00030002,
655 0x91d4, 0xffffffff, 0x00050004,
656 0x91e0, 0xffffffff, 0x00010006,
657 0x91e4, 0xffffffff, 0x00090008,
658 0x91e8, 0xffffffff, 0x00000000,
659 0x91ec, 0xffffffff, 0x00070000,
660 0x91f0, 0xffffffff, 0x00030002,
661 0x91f4, 0xffffffff, 0x00050004,
662 0x9200, 0xffffffff, 0x00010006,
663 0x9204, 0xffffffff, 0x00090008,
664 0x9208, 0xffffffff, 0x00070000,
665 0x920c, 0xffffffff, 0x00030002,
666 0x9210, 0xffffffff, 0x00050004,
667 0x921c, 0xffffffff, 0x00010006,
668 0x9220, 0xffffffff, 0x00090008,
669 0x9224, 0xffffffff, 0x00070000,
670 0x9228, 0xffffffff, 0x00030002,
671 0x922c, 0xffffffff, 0x00050004,
672 0x9238, 0xffffffff, 0x00010006,
673 0x923c, 0xffffffff, 0x00090008,
674 0x9240, 0xffffffff, 0x00070000,
675 0x9244, 0xffffffff, 0x00030002,
676 0x9248, 0xffffffff, 0x00050004,
677 0x9254, 0xffffffff, 0x00010006,
678 0x9258, 0xffffffff, 0x00090008,
679 0x925c, 0xffffffff, 0x00070000,
680 0x9260, 0xffffffff, 0x00030002,
681 0x9264, 0xffffffff, 0x00050004,
682 0x9270, 0xffffffff, 0x00010006,
683 0x9274, 0xffffffff, 0x00090008,
684 0x9278, 0xffffffff, 0x00070000,
685 0x927c, 0xffffffff, 0x00030002,
686 0x9280, 0xffffffff, 0x00050004,
687 0x928c, 0xffffffff, 0x00010006,
688 0x9290, 0xffffffff, 0x00090008,
689 0x9294, 0xffffffff, 0x00000000,
690 0x929c, 0xffffffff, 0x00000001,
691 0x802c, 0xffffffff, 0xc0000000,
692 0x977c, 0xffffffff, 0x00000100,
693 0x3f80, 0xffffffff, 0x00000100,
694 0xa210, 0xffffffff, 0x00000100,
695 0xa214, 0xffffffff, 0x00000100,
696 0x4d8, 0xffffffff, 0x00000100,
697 0x9784, 0xffffffff, 0x00000100,
698 0x9698, 0xffffffff, 0x00000100,
699 0x4d4, 0xffffffff, 0x00000200,
700 0x30cc, 0xffffffff, 0x00000100,
701 0x802c, 0xffffffff, 0xc0000000
702};
703
704static const u32 supersumo_golden_registers[] =
705{
706 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 707 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 708 0x7030, 0xffffffff, 0x00000011,
709 0x7c30, 0xffffffff, 0x00000011,
710 0x6104, 0x01000300, 0x00000000,
711 0x5bc0, 0x00300000, 0x00000000,
712 0x8c04, 0xffffffff, 0x40600060,
713 0x8c08, 0xffffffff, 0x001c001c,
714 0x8c20, 0xffffffff, 0x00800080,
715 0x8c24, 0xffffffff, 0x00800080,
716 0x8c18, 0xffffffff, 0x20202078,
717 0x8c1c, 0xffffffff, 0x00001010,
718 0x918c, 0xffffffff, 0x00010006,
719 0x91a8, 0xffffffff, 0x00010006,
720 0x91c4, 0xffffffff, 0x00010006,
721 0x91e0, 0xffffffff, 0x00010006,
722 0x9200, 0xffffffff, 0x00010006,
723 0x9150, 0xffffffff, 0x6e944040,
724 0x917c, 0xffffffff, 0x00030002,
725 0x9180, 0xffffffff, 0x00050004,
726 0x9198, 0xffffffff, 0x00030002,
727 0x919c, 0xffffffff, 0x00050004,
728 0x91b4, 0xffffffff, 0x00030002,
729 0x91b8, 0xffffffff, 0x00050004,
730 0x91d0, 0xffffffff, 0x00030002,
731 0x91d4, 0xffffffff, 0x00050004,
732 0x91f0, 0xffffffff, 0x00030002,
733 0x91f4, 0xffffffff, 0x00050004,
734 0x915c, 0xffffffff, 0x00010000,
735 0x9160, 0xffffffff, 0x00030002,
736 0x3f90, 0xffff0000, 0xff000000,
737 0x9178, 0xffffffff, 0x00070000,
738 0x9194, 0xffffffff, 0x00070000,
739 0x91b0, 0xffffffff, 0x00070000,
740 0x91cc, 0xffffffff, 0x00070000,
741 0x91ec, 0xffffffff, 0x00070000,
742 0x9148, 0xffff0000, 0xff000000,
743 0x9190, 0xffffffff, 0x00090008,
744 0x91ac, 0xffffffff, 0x00090008,
745 0x91c8, 0xffffffff, 0x00090008,
746 0x91e4, 0xffffffff, 0x00090008,
747 0x9204, 0xffffffff, 0x00090008,
748 0x3f94, 0xffff0000, 0xff000000,
749 0x914c, 0xffff0000, 0xff000000,
750 0x929c, 0xffffffff, 0x00000001,
751 0x8a18, 0xffffffff, 0x00000100,
752 0x8b28, 0xffffffff, 0x00000100,
753 0x9144, 0xffffffff, 0x00000100,
754 0x5644, 0xffffffff, 0x00000100,
755 0x9b7c, 0xffffffff, 0x00000000,
756 0x8030, 0xffffffff, 0x0000100a,
757 0x8a14, 0xffffffff, 0x00000007,
758 0x8b24, 0xffffffff, 0x00ff0fff,
759 0x8b10, 0xffffffff, 0x00000000,
760 0x28a4c, 0x06000000, 0x06000000,
761 0x4d8, 0xffffffff, 0x00000100,
762 0x913c, 0xffff000f, 0x0100000a,
763 0x960c, 0xffffffff, 0x54763210,
764 0x88c4, 0xffffffff, 0x000000c2,
765 0x88d4, 0xffffffff, 0x00000010,
766 0x8974, 0xffffffff, 0x00000000,
767 0xc78, 0x00000080, 0x00000080,
768 0x5e78, 0xffffffff, 0x001000f0,
769 0xd02c, 0xffffffff, 0x08421000,
770 0xa008, 0xffffffff, 0x00010000,
771 0x8d00, 0xffffffff, 0x100e4848,
772 0x8d04, 0xffffffff, 0x00164745,
773 0x8c00, 0xffffffff, 0xe4000003,
774 0x8cf0, 0x1fffffff, 0x08e00620,
775 0x28350, 0xffffffff, 0x00000000,
776 0x9508, 0xffffffff, 0x00000002
777};
778
779static const u32 sumo_golden_registers[] =
780{
781 0x900c, 0x00ffffff, 0x0017071f,
782 0x8c18, 0xffffffff, 0x10101060,
783 0x8c1c, 0xffffffff, 0x00001010,
784 0x8c30, 0x0000000f, 0x00000005,
785 0x9688, 0x0000000f, 0x00000007
786};
787
788static const u32 wrestler_golden_registers[] =
789{
790 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 791 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 792 0x7030, 0xffffffff, 0x00000011,
793 0x7c30, 0xffffffff, 0x00000011,
794 0x6104, 0x01000300, 0x00000000,
795 0x5bc0, 0x00300000, 0x00000000,
796 0x918c, 0xffffffff, 0x00010006,
797 0x91a8, 0xffffffff, 0x00010006,
798 0x9150, 0xffffffff, 0x6e944040,
799 0x917c, 0xffffffff, 0x00030002,
800 0x9198, 0xffffffff, 0x00030002,
801 0x915c, 0xffffffff, 0x00010000,
802 0x3f90, 0xffff0000, 0xff000000,
803 0x9178, 0xffffffff, 0x00070000,
804 0x9194, 0xffffffff, 0x00070000,
805 0x9148, 0xffff0000, 0xff000000,
806 0x9190, 0xffffffff, 0x00090008,
807 0x91ac, 0xffffffff, 0x00090008,
808 0x3f94, 0xffff0000, 0xff000000,
809 0x914c, 0xffff0000, 0xff000000,
810 0x929c, 0xffffffff, 0x00000001,
811 0x8a18, 0xffffffff, 0x00000100,
812 0x8b28, 0xffffffff, 0x00000100,
813 0x9144, 0xffffffff, 0x00000100,
814 0x9b7c, 0xffffffff, 0x00000000,
815 0x8030, 0xffffffff, 0x0000100a,
816 0x8a14, 0xffffffff, 0x00000001,
817 0x8b24, 0xffffffff, 0x00ff0fff,
818 0x8b10, 0xffffffff, 0x00000000,
819 0x28a4c, 0x06000000, 0x06000000,
820 0x4d8, 0xffffffff, 0x00000100,
821 0x913c, 0xffff000f, 0x0100000a,
822 0x960c, 0xffffffff, 0x54763210,
823 0x88c4, 0xffffffff, 0x000000c2,
824 0x88d4, 0xffffffff, 0x00000010,
825 0x8974, 0xffffffff, 0x00000000,
826 0xc78, 0x00000080, 0x00000080,
827 0x5e78, 0xffffffff, 0x001000f0,
828 0xd02c, 0xffffffff, 0x08421000,
829 0xa008, 0xffffffff, 0x00010000,
830 0x8d00, 0xffffffff, 0x100e4848,
831 0x8d04, 0xffffffff, 0x00164745,
832 0x8c00, 0xffffffff, 0xe4000003,
833 0x8cf0, 0x1fffffff, 0x08e00410,
834 0x28350, 0xffffffff, 0x00000000,
835 0x9508, 0xffffffff, 0x00000002,
836 0x900c, 0xffffffff, 0x0017071f,
837 0x8c18, 0xffffffff, 0x10101060,
838 0x8c1c, 0xffffffff, 0x00001010
839};
840
841static const u32 barts_golden_registers[] =
842{
843 0x5eb4, 0xffffffff, 0x00000002,
844 0x5e78, 0x8f311ff1, 0x001000f0,
845 0x3f90, 0xffff0000, 0xff000000,
846 0x9148, 0xffff0000, 0xff000000,
847 0x3f94, 0xffff0000, 0xff000000,
848 0x914c, 0xffff0000, 0xff000000,
849 0xc78, 0x00000080, 0x00000080,
850 0xbd4, 0x70073777, 0x00010001,
851 0xd02c, 0xbfffff1f, 0x08421000,
852 0xd0b8, 0x03773777, 0x02011003,
853 0x5bc0, 0x00200000, 0x50100000,
854 0x98f8, 0x33773777, 0x02011003,
855 0x98fc, 0xffffffff, 0x76543210,
856 0x7030, 0x31000311, 0x00000011,
857 0x2f48, 0x00000007, 0x02011003,
858 0x6b28, 0x00000010, 0x00000012,
859 0x7728, 0x00000010, 0x00000012,
860 0x10328, 0x00000010, 0x00000012,
861 0x10f28, 0x00000010, 0x00000012,
862 0x11b28, 0x00000010, 0x00000012,
863 0x12728, 0x00000010, 0x00000012,
864 0x240c, 0x000007ff, 0x00000380,
865 0x8a14, 0xf000001f, 0x00000007,
866 0x8b24, 0x3fff3fff, 0x00ff0fff,
867 0x8b10, 0x0000ff0f, 0x00000000,
868 0x28a4c, 0x07ffffff, 0x06000000,
869 0x10c, 0x00000001, 0x00010003,
870 0xa02c, 0xffffffff, 0x0000009b,
871 0x913c, 0x0000000f, 0x0100000a,
872 0x8d00, 0xffff7f7f, 0x100e4848,
873 0x8d04, 0x00ffffff, 0x00164745,
874 0x8c00, 0xfffc0003, 0xe4000003,
875 0x8c04, 0xf8ff00ff, 0x40600060,
876 0x8c08, 0x00ff00ff, 0x001c001c,
877 0x8cf0, 0x1fff1fff, 0x08e00620,
878 0x8c20, 0x0fff0fff, 0x00800080,
879 0x8c24, 0x0fff0fff, 0x00800080,
880 0x8c18, 0xffffffff, 0x20202078,
881 0x8c1c, 0x0000ffff, 0x00001010,
882 0x28350, 0x00000f01, 0x00000000,
883 0x9508, 0x3700001f, 0x00000002,
884 0x960c, 0xffffffff, 0x54763210,
885 0x88c4, 0x001f3ae3, 0x000000c2,
886 0x88d4, 0x0000001f, 0x00000010,
887 0x8974, 0xffffffff, 0x00000000
888};
889
890static const u32 turks_golden_registers[] =
891{
892 0x5eb4, 0xffffffff, 0x00000002,
893 0x5e78, 0x8f311ff1, 0x001000f0,
894 0x8c8, 0x00003000, 0x00001070,
895 0x8cc, 0x000fffff, 0x00040035,
896 0x3f90, 0xffff0000, 0xfff00000,
897 0x9148, 0xffff0000, 0xfff00000,
898 0x3f94, 0xffff0000, 0xfff00000,
899 0x914c, 0xffff0000, 0xfff00000,
900 0xc78, 0x00000080, 0x00000080,
901 0xbd4, 0x00073007, 0x00010002,
902 0xd02c, 0xbfffff1f, 0x08421000,
903 0xd0b8, 0x03773777, 0x02010002,
904 0x5bc0, 0x00200000, 0x50100000,
905 0x98f8, 0x33773777, 0x00010002,
906 0x98fc, 0xffffffff, 0x33221100,
907 0x7030, 0x31000311, 0x00000011,
908 0x2f48, 0x33773777, 0x00010002,
909 0x6b28, 0x00000010, 0x00000012,
910 0x7728, 0x00000010, 0x00000012,
911 0x10328, 0x00000010, 0x00000012,
912 0x10f28, 0x00000010, 0x00000012,
913 0x11b28, 0x00000010, 0x00000012,
914 0x12728, 0x00000010, 0x00000012,
915 0x240c, 0x000007ff, 0x00000380,
916 0x8a14, 0xf000001f, 0x00000007,
917 0x8b24, 0x3fff3fff, 0x00ff0fff,
918 0x8b10, 0x0000ff0f, 0x00000000,
919 0x28a4c, 0x07ffffff, 0x06000000,
920 0x10c, 0x00000001, 0x00010003,
921 0xa02c, 0xffffffff, 0x0000009b,
922 0x913c, 0x0000000f, 0x0100000a,
923 0x8d00, 0xffff7f7f, 0x100e4848,
924 0x8d04, 0x00ffffff, 0x00164745,
925 0x8c00, 0xfffc0003, 0xe4000003,
926 0x8c04, 0xf8ff00ff, 0x40600060,
927 0x8c08, 0x00ff00ff, 0x001c001c,
928 0x8cf0, 0x1fff1fff, 0x08e00410,
929 0x8c20, 0x0fff0fff, 0x00800080,
930 0x8c24, 0x0fff0fff, 0x00800080,
931 0x8c18, 0xffffffff, 0x20202078,
932 0x8c1c, 0x0000ffff, 0x00001010,
933 0x28350, 0x00000f01, 0x00000000,
934 0x9508, 0x3700001f, 0x00000002,
935 0x960c, 0xffffffff, 0x54763210,
936 0x88c4, 0x001f3ae3, 0x000000c2,
937 0x88d4, 0x0000001f, 0x00000010,
938 0x8974, 0xffffffff, 0x00000000
939};
940
941static const u32 caicos_golden_registers[] =
942{
943 0x5eb4, 0xffffffff, 0x00000002,
944 0x5e78, 0x8f311ff1, 0x001000f0,
945 0x8c8, 0x00003420, 0x00001450,
946 0x8cc, 0x000fffff, 0x00040035,
947 0x3f90, 0xffff0000, 0xfffc0000,
948 0x9148, 0xffff0000, 0xfffc0000,
949 0x3f94, 0xffff0000, 0xfffc0000,
950 0x914c, 0xffff0000, 0xfffc0000,
951 0xc78, 0x00000080, 0x00000080,
952 0xbd4, 0x00073007, 0x00010001,
953 0xd02c, 0xbfffff1f, 0x08421000,
954 0xd0b8, 0x03773777, 0x02010001,
955 0x5bc0, 0x00200000, 0x50100000,
956 0x98f8, 0x33773777, 0x02010001,
957 0x98fc, 0xffffffff, 0x33221100,
958 0x7030, 0x31000311, 0x00000011,
959 0x2f48, 0x33773777, 0x02010001,
960 0x6b28, 0x00000010, 0x00000012,
961 0x7728, 0x00000010, 0x00000012,
962 0x10328, 0x00000010, 0x00000012,
963 0x10f28, 0x00000010, 0x00000012,
964 0x11b28, 0x00000010, 0x00000012,
965 0x12728, 0x00000010, 0x00000012,
966 0x240c, 0x000007ff, 0x00000380,
967 0x8a14, 0xf000001f, 0x00000001,
968 0x8b24, 0x3fff3fff, 0x00ff0fff,
969 0x8b10, 0x0000ff0f, 0x00000000,
970 0x28a4c, 0x07ffffff, 0x06000000,
971 0x10c, 0x00000001, 0x00010003,
972 0xa02c, 0xffffffff, 0x0000009b,
973 0x913c, 0x0000000f, 0x0100000a,
974 0x8d00, 0xffff7f7f, 0x100e4848,
975 0x8d04, 0x00ffffff, 0x00164745,
976 0x8c00, 0xfffc0003, 0xe4000003,
977 0x8c04, 0xf8ff00ff, 0x40600060,
978 0x8c08, 0x00ff00ff, 0x001c001c,
979 0x8cf0, 0x1fff1fff, 0x08e00410,
980 0x8c20, 0x0fff0fff, 0x00800080,
981 0x8c24, 0x0fff0fff, 0x00800080,
982 0x8c18, 0xffffffff, 0x20202078,
983 0x8c1c, 0x0000ffff, 0x00001010,
984 0x28350, 0x00000f01, 0x00000000,
985 0x9508, 0x3700001f, 0x00000002,
986 0x960c, 0xffffffff, 0x54763210,
987 0x88c4, 0x001f3ae3, 0x000000c2,
988 0x88d4, 0x0000001f, 0x00000010,
989 0x8974, 0xffffffff, 0x00000000
990};
991
992static void evergreen_init_golden_registers(struct radeon_device *rdev)
993{
994 switch (rdev->family) {
995 case CHIP_CYPRESS:
996 case CHIP_HEMLOCK:
997 radeon_program_register_sequence(rdev,
998 evergreen_golden_registers,
999 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1000 radeon_program_register_sequence(rdev,
1001 evergreen_golden_registers2,
1002 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1003 radeon_program_register_sequence(rdev,
1004 cypress_mgcg_init,
1005 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1006 break;
1007 case CHIP_JUNIPER:
1008 radeon_program_register_sequence(rdev,
1009 evergreen_golden_registers,
1010 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1011 radeon_program_register_sequence(rdev,
1012 evergreen_golden_registers2,
1013 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1014 radeon_program_register_sequence(rdev,
1015 juniper_mgcg_init,
1016 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1017 break;
1018 case CHIP_REDWOOD:
1019 radeon_program_register_sequence(rdev,
1020 evergreen_golden_registers,
1021 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1022 radeon_program_register_sequence(rdev,
1023 evergreen_golden_registers2,
1024 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1025 radeon_program_register_sequence(rdev,
1026 redwood_mgcg_init,
1027 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1028 break;
1029 case CHIP_CEDAR:
1030 radeon_program_register_sequence(rdev,
1031 cedar_golden_registers,
1032 (const u32)ARRAY_SIZE(cedar_golden_registers));
1033 radeon_program_register_sequence(rdev,
1034 evergreen_golden_registers2,
1035 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1036 radeon_program_register_sequence(rdev,
1037 cedar_mgcg_init,
1038 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1039 break;
1040 case CHIP_PALM:
1041 radeon_program_register_sequence(rdev,
1042 wrestler_golden_registers,
1043 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1044 break;
1045 case CHIP_SUMO:
1046 radeon_program_register_sequence(rdev,
1047 supersumo_golden_registers,
1048 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1049 break;
1050 case CHIP_SUMO2:
1051 radeon_program_register_sequence(rdev,
1052 supersumo_golden_registers,
1053 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054 radeon_program_register_sequence(rdev,
1055 sumo_golden_registers,
1056 (const u32)ARRAY_SIZE(sumo_golden_registers));
1057 break;
1058 case CHIP_BARTS:
1059 radeon_program_register_sequence(rdev,
1060 barts_golden_registers,
1061 (const u32)ARRAY_SIZE(barts_golden_registers));
1062 break;
1063 case CHIP_TURKS:
1064 radeon_program_register_sequence(rdev,
1065 turks_golden_registers,
1066 (const u32)ARRAY_SIZE(turks_golden_registers));
1067 break;
1068 case CHIP_CAICOS:
1069 radeon_program_register_sequence(rdev,
1070 caicos_golden_registers,
1071 (const u32)ARRAY_SIZE(caicos_golden_registers));
1072 break;
1073 default:
1074 break;
1075 }
1076}
1077
ff609975
AD		 1078/**
1079 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1080 *
1081 * @rdev: radeon_device pointer
1082 * @reg: register offset in bytes
1083 * @val: register value
1084 *
1085 * Returns 0 for success or -EINVAL for an invalid register
1086 *
1087 */
1088int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1089 u32 reg, u32 *val)
1090{
1091 switch (reg) {
1092 case GRBM_STATUS:
1093 case GRBM_STATUS_SE0:
1094 case GRBM_STATUS_SE1:
1095 case SRBM_STATUS:
1096 case SRBM_STATUS2:
1097 case DMA_STATUS_REG:
1098 case UVD_STATUS:
1099 *val = RREG32(reg);
1100 return 0;
1101 default:
1102 return -EINVAL;
1103 }
1104}
1105
285484e2
JG		 1106void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1107 unsigned *bankh, unsigned *mtaspect,
1108 unsigned *tile_split)
1109{
1110 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1111 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1112 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1113 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1114 switch (*bankw) {
1115 default:
1116 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1117 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1118 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1119 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1120 }
1121 switch (*bankh) {
1122 default:
1123 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1124 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1125 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1126 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1127 }
1128 switch (*mtaspect) {
1129 default:
1130 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1131 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1132 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1133 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1134 }
1135}
1136
23d33ba3
AD		 1137static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1138 u32 cntl_reg, u32 status_reg)
1139{
1140 int r, i;
1141 struct atom_clock_dividers dividers;
1142
1143 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1144 clock, false, &dividers);
1145 if (r)
1146 return r;
1147
1148 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1149
1150 for (i = 0; i < 100; i++) {
1151 if (RREG32(status_reg) & DCLK_STATUS)
1152 break;
1153 mdelay(10);
1154 }
1155 if (i == 100)
1156 return -ETIMEDOUT;
1157
1158 return 0;
1159}
1160
1161int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1162{
1163 int r = 0;
1164 u32 cg_scratch = RREG32(CG_SCRATCH1);
1165
1166 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1167 if (r)
1168 goto done;
1169 cg_scratch &= 0xffff0000;
1170 cg_scratch |= vclk / 100; /* Mhz */
1171
1172 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1173 if (r)
1174 goto done;
1175 cg_scratch &= 0x0000ffff;
1176 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1177
1178done:
1179 WREG32(CG_SCRATCH1, cg_scratch);
1180
1181 return r;
1182}
1183
a8b4925c
AD		 1184int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1185{
1186 /* start off with something large */
facd112d 1187 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1188 int r;
1189
4ed10835
CK		 1190 /* bypass vclk and dclk with bclk */
1191 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1192 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1193 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1194
1195 /* put PLL in bypass mode */
1196 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1197
1198 if (!vclk || !dclk) {
1199 /* keep the Bypass mode, put PLL to sleep */
1200 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1201 return 0;
1202 }
1203
facd112d
CK		 1204 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1205 16384, 0x03FFFFFF, 0, 128, 5,
1206 &fb_div, &vclk_div, &dclk_div);
1207 if (r)
1208 return r;
a8b4925c
AD		 1209
1210 /* set VCO_MODE to 1 */
1211 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1212
1213 /* toggle UPLL_SLEEP to 1 then back to 0 */
1214 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1215 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1216
1217 /* deassert UPLL_RESET */
1218 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1219
1220 mdelay(1);
1221
facd112d 1222 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1223 if (r)
1224 return r;
1225
1226 /* assert UPLL_RESET again */
1227 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1228
1229 /* disable spread spectrum. */
1230 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1231
1232 /* set feedback divider */
facd112d 1233 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1234
1235 /* set ref divider to 0 */
1236 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1237
facd112d 1238 if (fb_div < 307200)
a8b4925c
AD		 1239 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1240 else
1241 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1242
1243 /* set PDIV_A and PDIV_B */
1244 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1245 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1246 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1247
1248 /* give the PLL some time to settle */
1249 mdelay(15);
1250
1251 /* deassert PLL_RESET */
1252 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1253
1254 mdelay(15);
1255
1256 /* switch from bypass mode to normal mode */
1257 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1258
facd112d 1259 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1260 if (r)
1261 return r;
1262
1263 /* switch VCLK and DCLK selection */
1264 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1265 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1266 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1267
1268 mdelay(100);
1269
1270 return 0;
1271}
1272
d054ac16
AD		 1273void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1274{
c11592fe
YW		 1275 int readrq;
1276 u16 v;
d054ac16 1277
c11592fe
YW		 1278 readrq = pcie_get_readrq(rdev->pdev);
1279 v = ffs(readrq) - 8;
d054ac16
AD		 1280 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1281 * to avoid hangs or perfomance issues
1282 */
c11592fe
YW		 1283 if ((v == 0) || (v == 6) || (v == 7))
1284 pcie_set_readrq(rdev->pdev, 512);
d054ac16
AD		 1285}
1286
134b480f
AD		 1287void dce4_program_fmt(struct drm_encoder *encoder)
1288{
1289 struct drm_device *dev = encoder->dev;
1290 struct radeon_device *rdev = dev->dev_private;
1291 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1292 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1293 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1294 int bpc = 0;
1295 u32 tmp = 0;
6214bb74 1296 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 1297
6214bb74
AD		 1298 if (connector) {
1299 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 1300 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 1301 dither = radeon_connector->dither;
1302 }
134b480f
AD		 1303
1304 /* LVDS/eDP FMT is set up by atom */
1305 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1306 return;
1307
1308 /* not needed for analog */
1309 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1310 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1311 return;
1312
1313 if (bpc == 0)
1314 return;
1315
1316 switch (bpc) {
1317 case 6:
6214bb74 1318 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1319 /* XXX sort out optimal dither settings */
1320 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1321 FMT_SPATIAL_DITHER_EN);
1322 else
1323 tmp |= FMT_TRUNCATE_EN;
1324 break;
1325 case 8:
6214bb74 1326 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1327 /* XXX sort out optimal dither settings */
1328 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329 FMT_RGB_RANDOM_ENABLE |
1330 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1331 else
1332 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1333 break;
1334 case 10:
1335 default:
1336 /* not needed */
1337 break;
1338 }
1339
1340 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1341}
1342
10257a6d
AD		 1343static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1344{
1345 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1346 return true;
1347 else
1348 return false;
1349}
1350
1351static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1352{
1353 u32 pos1, pos2;
1354
1355 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1356 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1357
1358 if (pos1 != pos2)
1359 return true;
1360 else
1361 return false;
1362}
1363
377edc8b
AD		 1364/**
1365 * dce4_wait_for_vblank - vblank wait asic callback.
1366 *
1367 * @rdev: radeon_device pointer
1368 * @crtc: crtc to wait for vblank on
1369 *
1370 * Wait for vblank on the requested crtc (evergreen+).
1371 */
3ae19b75
AD		 1372void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1373{
10257a6d 1374 unsigned i = 0;
3ae19b75 1375
4a15903d
AD		 1376 if (crtc >= rdev->num_crtc)
1377 return;
1378
10257a6d
AD		 1379 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1380 return;
1381
1382 /* depending on when we hit vblank, we may be close to active; if so,
1383 * wait for another frame.
1384 */
1385 while (dce4_is_in_vblank(rdev, crtc)) {
1386 if (i++ % 100 == 0) {
1387 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1388 break;
3ae19b75 1389 }
10257a6d
AD		 1390 }
1391
1392 while (!dce4_is_in_vblank(rdev, crtc)) {
1393 if (i++ % 100 == 0) {
1394 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1395 break;
3ae19b75
AD		 1396 }
1397 }
1398}
1399
377edc8b
AD		 1400/**
1401 * evergreen_page_flip - pageflip callback.
1402 *
1403 * @rdev: radeon_device pointer
1404 * @crtc_id: crtc to cleanup pageflip on
1405 * @crtc_base: new address of the crtc (GPU MC address)
1406 *
cbd3e249
AD		 1407 * Triggers the actual pageflip by updating the primary
1408 * surface base address (evergreen+).
377edc8b 1409 */
157fa14d 1410void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
6f34be50
AD		 1411{
1412 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
6f34be50
AD		 1413
1414 /* update the scanout addresses */
6f34be50
AD		 1415 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1416 upper_32_bits(crtc_base));
1417 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1418 (u32)crtc_base);
cbd3e249
AD		 1419 /* post the write */
1420 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
157fa14d
CK		 1421}
1422
1423/**
1424 * evergreen_page_flip_pending - check if page flip is still pending
1425 *
1426 * @rdev: radeon_device pointer
1427 * @crtc_id: crtc to check
1428 *
1429 * Returns the current update pending status.
1430 */
1431bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1432{
1433 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
6f34be50
AD		 1434
1435 /* Return current update_pending status: */
157fa14d
CK		 1436 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1437 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
6f34be50
AD		 1438}
1439
21a8122a 1440/* get temperature in millidegrees */
20d391d7 1441int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1442{
1c88d74f
AD		 1443 u32 temp, toffset;
1444 int actual_temp = 0;
67b3f823
AD		 1445
1446 if (rdev->family == CHIP_JUNIPER) {
1447 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1448 TOFFSET_SHIFT;
1449 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1450 TS0_ADC_DOUT_SHIFT;
1451
1452 if (toffset & 0x100)
1453 actual_temp = temp / 2 - (0x200 - toffset);
1454 else
1455 actual_temp = temp / 2 + toffset;
1456
1457 actual_temp = actual_temp * 1000;
1458
1459 } else {
1460 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1461 ASIC_T_SHIFT;
1462
1463 if (temp & 0x400)
1464 actual_temp = -256;
1465 else if (temp & 0x200)
1466 actual_temp = 255;
1467 else if (temp & 0x100) {
1468 actual_temp = temp & 0x1ff;
1469 actual_temp |= ~0x1ff;
1470 } else
1471 actual_temp = temp & 0xff;
1472
1473 actual_temp = (actual_temp * 1000) / 2;
1474 }
21a8122a 1475
67b3f823 1476 return actual_temp;
21a8122a
AD		 1477}
1478
20d391d7 1479int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1480{
1481 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1482 int actual_temp = temp - 49;
e33df25f
AD		 1483
1484 return actual_temp * 1000;
1485}
1486
377edc8b
AD		 1487/**
1488 * sumo_pm_init_profile - Initialize power profiles callback.
1489 *
1490 * @rdev: radeon_device pointer
1491 *
1492 * Initialize the power states used in profile mode
1493 * (sumo, trinity, SI).
1494 * Used for profile mode only.
1495 */
a4c9e2ee
AD		 1496void sumo_pm_init_profile(struct radeon_device *rdev)
1497{
1498 int idx;
1499
1500 /* default */
1501 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1502 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1503 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1504 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1505
1506 /* low,mid sh/mh */
1507 if (rdev->flags & RADEON_IS_MOBILITY)
1508 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1509 else
1510 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1511
1512 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1513 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1514 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1515 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1516
1517 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1518 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1519 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1520 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1521
1522 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1523 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1524 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1525 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1526
1527 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1529 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1530 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1531
1532 /* high sh/mh */
1533 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1534 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1535 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1536 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1537 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1538 rdev->pm.power_state[idx].num_clock_modes - 1;
1539
1540 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1541 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1543 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1544 rdev->pm.power_state[idx].num_clock_modes - 1;
1545}
1546
27810fb2
AD		 1547/**
1548 * btc_pm_init_profile - Initialize power profiles callback.
1549 *
1550 * @rdev: radeon_device pointer
1551 *
1552 * Initialize the power states used in profile mode
1553 * (BTC, cayman).
1554 * Used for profile mode only.
1555 */
1556void btc_pm_init_profile(struct radeon_device *rdev)
1557{
1558 int idx;
1559
1560 /* default */
1561 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1562 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1563 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1564 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1565 /* starting with BTC, there is one state that is used for both
1566 * MH and SH. Difference is that we always use the high clock index for
1567 * mclk.
1568 */
1569 if (rdev->flags & RADEON_IS_MOBILITY)
1570 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1571 else
1572 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1573 /* low sh */
1574 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1575 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1576 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1577 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1578 /* mid sh */
1579 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1580 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1581 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1582 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1583 /* high sh */
1584 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1585 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1586 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1587 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1588 /* low mh */
1589 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1590 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1591 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1592 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1593 /* mid mh */
1594 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1595 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1596 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1597 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1598 /* high mh */
1599 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1600 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1601 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1602 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1603}
1604
377edc8b
AD		 1605/**
1606 * evergreen_pm_misc - set additional pm hw parameters callback.
1607 *
1608 * @rdev: radeon_device pointer
1609 *
1610 * Set non-clock parameters associated with a power state
1611 * (voltage, etc.) (evergreen+).
1612 */
49e02b73
AD		 1613void evergreen_pm_misc(struct radeon_device *rdev)
1614{
a081a9d6
RM		 1615 int req_ps_idx = rdev->pm.requested_power_state_index;
1616 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1617 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1618 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1619
2feea49a 1620 if (voltage->type == VOLTAGE_SW) {
c6cf7777
AD		 1621 /* 0xff0x are flags rather then an actual voltage */
1622 if ((voltage->voltage & 0xff00) == 0xff00)
a377e187 1623 return;
2feea49a 1624 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1625 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1626 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1627 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1628 }
7ae764b1
AD		 1629
1630 /* starting with BTC, there is one state that is used for both
1631 * MH and SH. Difference is that we always use the high clock index for
1632 * mclk and vddci.
1633 */
1634 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1635 (rdev->family >= CHIP_BARTS) &&
1636 rdev->pm.active_crtc_count &&
1637 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1638 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1639 voltage = &rdev->pm.power_state[req_ps_idx].
1640 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1641
c6cf7777
AD		 1642 /* 0xff0x are flags rather then an actual voltage */
1643 if ((voltage->vddci & 0xff00) == 0xff00)
a377e187 1644 return;
2feea49a
AD		 1645 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1646 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1647 rdev->pm.current_vddci = voltage->vddci;
1648 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1649 }
1650 }
49e02b73
AD		 1651}
1652
377edc8b
AD		 1653/**
1654 * evergreen_pm_prepare - pre-power state change callback.
1655 *
1656 * @rdev: radeon_device pointer
1657 *
1658 * Prepare for a power state change (evergreen+).
1659 */
49e02b73
AD		 1660void evergreen_pm_prepare(struct radeon_device *rdev)
1661{
1662 struct drm_device *ddev = rdev->ddev;
1663 struct drm_crtc *crtc;
1664 struct radeon_crtc *radeon_crtc;
1665 u32 tmp;
1666
1667 /* disable any active CRTCs */
1668 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1669 radeon_crtc = to_radeon_crtc(crtc);
1670 if (radeon_crtc->enabled) {
1671 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1672 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1673 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1674 }
1675 }
1676}
1677
377edc8b
AD		 1678/**
1679 * evergreen_pm_finish - post-power state change callback.
1680 *
1681 * @rdev: radeon_device pointer
1682 *
1683 * Clean up after a power state change (evergreen+).
1684 */
49e02b73
AD		 1685void evergreen_pm_finish(struct radeon_device *rdev)
1686{
1687 struct drm_device *ddev = rdev->ddev;
1688 struct drm_crtc *crtc;
1689 struct radeon_crtc *radeon_crtc;
1690 u32 tmp;
1691
1692 /* enable any active CRTCs */
1693 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1694 radeon_crtc = to_radeon_crtc(crtc);
1695 if (radeon_crtc->enabled) {
1696 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1697 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1698 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1699 }
1700 }
1701}
1702
377edc8b
AD		 1703/**
1704 * evergreen_hpd_sense - hpd sense callback.
1705 *
1706 * @rdev: radeon_device pointer
1707 * @hpd: hpd (hotplug detect) pin
1708 *
1709 * Checks if a digital monitor is connected (evergreen+).
1710 * Returns true if connected, false if not connected.
1711 */
bcc1c2a1
AD		 1712bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1713{
1714 bool connected = false;
0ca2ab52
AD		 1715
1716 switch (hpd) {
1717 case RADEON_HPD_1:
1718 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1719 connected = true;
1720 break;
1721 case RADEON_HPD_2:
1722 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1723 connected = true;
1724 break;
1725 case RADEON_HPD_3:
1726 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1727 connected = true;
1728 break;
1729 case RADEON_HPD_4:
1730 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1731 connected = true;
1732 break;
1733 case RADEON_HPD_5:
1734 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1735 connected = true;
1736 break;
1737 case RADEON_HPD_6:
1738 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1739 connected = true;
ebc54ffe 1740 break;
0ca2ab52
AD		 1741 default:
1742 break;
1743 }
1744
bcc1c2a1
AD		 1745 return connected;
1746}
1747
377edc8b
AD		 1748/**
1749 * evergreen_hpd_set_polarity - hpd set polarity callback.
1750 *
1751 * @rdev: radeon_device pointer
1752 * @hpd: hpd (hotplug detect) pin
1753 *
1754 * Set the polarity of the hpd pin (evergreen+).
1755 */
bcc1c2a1
AD		 1756void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1757 enum radeon_hpd_id hpd)
1758{
0ca2ab52
AD		 1759 u32 tmp;
1760 bool connected = evergreen_hpd_sense(rdev, hpd);
1761
1762 switch (hpd) {
1763 case RADEON_HPD_1:
1764 tmp = RREG32(DC_HPD1_INT_CONTROL);
1765 if (connected)
1766 tmp &= ~DC_HPDx_INT_POLARITY;
1767 else
1768 tmp |= DC_HPDx_INT_POLARITY;
1769 WREG32(DC_HPD1_INT_CONTROL, tmp);
1770 break;
1771 case RADEON_HPD_2:
1772 tmp = RREG32(DC_HPD2_INT_CONTROL);
1773 if (connected)
1774 tmp &= ~DC_HPDx_INT_POLARITY;
1775 else
1776 tmp |= DC_HPDx_INT_POLARITY;
1777 WREG32(DC_HPD2_INT_CONTROL, tmp);
1778 break;
1779 case RADEON_HPD_3:
1780 tmp = RREG32(DC_HPD3_INT_CONTROL);
1781 if (connected)
1782 tmp &= ~DC_HPDx_INT_POLARITY;
1783 else
1784 tmp |= DC_HPDx_INT_POLARITY;
1785 WREG32(DC_HPD3_INT_CONTROL, tmp);
1786 break;
1787 case RADEON_HPD_4:
1788 tmp = RREG32(DC_HPD4_INT_CONTROL);
1789 if (connected)
1790 tmp &= ~DC_HPDx_INT_POLARITY;
1791 else
1792 tmp |= DC_HPDx_INT_POLARITY;
1793 WREG32(DC_HPD4_INT_CONTROL, tmp);
1794 break;
1795 case RADEON_HPD_5:
1796 tmp = RREG32(DC_HPD5_INT_CONTROL);
1797 if (connected)
1798 tmp &= ~DC_HPDx_INT_POLARITY;
1799 else
1800 tmp |= DC_HPDx_INT_POLARITY;
1801 WREG32(DC_HPD5_INT_CONTROL, tmp);
1802 break;
1803 case RADEON_HPD_6:
1804 tmp = RREG32(DC_HPD6_INT_CONTROL);
1805 if (connected)
1806 tmp &= ~DC_HPDx_INT_POLARITY;
1807 else
1808 tmp |= DC_HPDx_INT_POLARITY;
1809 WREG32(DC_HPD6_INT_CONTROL, tmp);
1810 break;
1811 default:
1812 break;
1813 }
bcc1c2a1
AD		 1814}
1815
377edc8b
AD		 1816/**
1817 * evergreen_hpd_init - hpd setup callback.
1818 *
1819 * @rdev: radeon_device pointer
1820 *
1821 * Setup the hpd pins used by the card (evergreen+).
1822 * Enable the pin, set the polarity, and enable the hpd interrupts.
1823 */
bcc1c2a1
AD		 1824void evergreen_hpd_init(struct radeon_device *rdev)
1825{
0ca2ab52
AD		 1826 struct drm_device *dev = rdev->ddev;
1827 struct drm_connector *connector;
fb98257a 1828 unsigned enabled = 0;
0ca2ab52
AD		 1829 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1830 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1831
0ca2ab52
AD		 1832 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1833 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1834
1835 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1836 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1837 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1838 * aux dp channel on imac and help (but not completely fix)
1839 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1840 * also avoid interrupt storms during dpms.
1841 */
1842 continue;
1843 }
0ca2ab52
AD		 1844 switch (radeon_connector->hpd.hpd) {
1845 case RADEON_HPD_1:
1846 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1847 break;
1848 case RADEON_HPD_2:
1849 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1850 break;
1851 case RADEON_HPD_3:
1852 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1853 break;
1854 case RADEON_HPD_4:
1855 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1856 break;
1857 case RADEON_HPD_5:
1858 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1859 break;
1860 case RADEON_HPD_6:
1861 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1862 break;
1863 default:
1864 break;
1865 }
64912e99 1866 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1867 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1868 }
fb98257a 1869 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1870}
1871
377edc8b
AD		 1872/**
1873 * evergreen_hpd_fini - hpd tear down callback.
1874 *
1875 * @rdev: radeon_device pointer
1876 *
1877 * Tear down the hpd pins used by the card (evergreen+).
1878 * Disable the hpd interrupts.
1879 */
0ca2ab52 1880void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1881{
0ca2ab52
AD		 1882 struct drm_device *dev = rdev->ddev;
1883 struct drm_connector *connector;
fb98257a 1884 unsigned disabled = 0;
0ca2ab52
AD		 1885
1886 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1887 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1888 switch (radeon_connector->hpd.hpd) {
1889 case RADEON_HPD_1:
1890 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1891 break;
1892 case RADEON_HPD_2:
1893 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1894 break;
1895 case RADEON_HPD_3:
1896 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1897 break;
1898 case RADEON_HPD_4:
1899 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1900 break;
1901 case RADEON_HPD_5:
1902 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1903 break;
1904 case RADEON_HPD_6:
1905 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1906 break;
1907 default:
1908 break;
1909 }
fb98257a 1910 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1911 }
fb98257a 1912 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1913}
1914
f9d9c362
AD		 1915/* watermark setup */
1916
1917static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1918 struct radeon_crtc *radeon_crtc,
1919 struct drm_display_mode *mode,
1920 struct drm_display_mode *other_mode)
1921{
0b31e023
AD		 1922 u32 tmp, buffer_alloc, i;
1923 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
f9d9c362
AD		 1924 /*
1925 * Line Buffer Setup
1926 * There are 3 line buffers, each one shared by 2 display controllers.
1927 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1928 * the display controllers. The paritioning is done via one of four
1929 * preset allocations specified in bits 2:0:
1930 * first display controller
1931 * 0 - first half of lb (3840 * 2)
1932 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1933 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1934 * 3 - first 1/4 of lb (1920 * 2)
1935 * second display controller
1936 * 4 - second half of lb (3840 * 2)
1937 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1938 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1939 * 7 - last 1/4 of lb (1920 * 2)
1940 */
12dfc843
AD		 1941 /* this can get tricky if we have two large displays on a paired group
1942 * of crtcs. Ideally for multiple large displays we'd assign them to
1943 * non-linked crtcs for maximum line buffer allocation.
1944 */
1945 if (radeon_crtc->base.enabled && mode) {
0b31e023 1946 if (other_mode) {
f9d9c362 1947 tmp = 0; /* 1/2 */
0b31e023
AD		 1948 buffer_alloc = 1;
1949 } else {
12dfc843 1950 tmp = 2; /* whole */
0b31e023
AD		 1951 buffer_alloc = 2;
1952 }
1953 } else {
12dfc843 1954 tmp = 0;
0b31e023
AD		 1955 buffer_alloc = 0;
1956 }
f9d9c362
AD		 1957
1958 /* second controller of the pair uses second half of the lb */
1959 if (radeon_crtc->crtc_id % 2)
1960 tmp += 4;
1961 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1962
0b31e023
AD		 1963 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1964 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1965 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1966 for (i = 0; i < rdev->usec_timeout; i++) {
1967 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1968 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1969 break;
1970 udelay(1);
1971 }
1972 }
1973
12dfc843
AD		 1974 if (radeon_crtc->base.enabled && mode) {
1975 switch (tmp) {
1976 case 0:
1977 case 4:
1978 default:
1979 if (ASIC_IS_DCE5(rdev))
1980 return 4096 * 2;
1981 else
1982 return 3840 * 2;
1983 case 1:
1984 case 5:
1985 if (ASIC_IS_DCE5(rdev))
1986 return 6144 * 2;
1987 else
1988 return 5760 * 2;
1989 case 2:
1990 case 6:
1991 if (ASIC_IS_DCE5(rdev))
1992 return 8192 * 2;
1993 else
1994 return 7680 * 2;
1995 case 3:
1996 case 7:
1997 if (ASIC_IS_DCE5(rdev))
1998 return 2048 * 2;
1999 else
2000 return 1920 * 2;
2001 }
f9d9c362 2002 }
12dfc843
AD		 2003
2004 /* controller not enabled, so no lb used */
2005 return 0;
f9d9c362
AD		 2006}
2007
ca7db22b 2008u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 2009{
2010 u32 tmp = RREG32(MC_SHARED_CHMAP);
2011
2012 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2013 case 0:
2014 default:
2015 return 1;
2016 case 1:
2017 return 2;
2018 case 2:
2019 return 4;
2020 case 3:
2021 return 8;
2022 }
2023}
2024
2025struct evergreen_wm_params {
2026 u32 dram_channels; /* number of dram channels */
2027 u32 yclk; /* bandwidth per dram data pin in kHz */
2028 u32 sclk; /* engine clock in kHz */
2029 u32 disp_clk; /* display clock in kHz */
2030 u32 src_width; /* viewport width */
2031 u32 active_time; /* active display time in ns */
2032 u32 blank_time; /* blank time in ns */
2033 bool interlaced; /* mode is interlaced */
2034 fixed20_12 vsc; /* vertical scale ratio */
2035 u32 num_heads; /* number of active crtcs */
2036 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2037 u32 lb_size; /* line buffer allocated to pipe */
2038 u32 vtaps; /* vertical scaler taps */
2039};
2040
2041static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
2042{
2043 /* Calculate DRAM Bandwidth and the part allocated to display. */
2044 fixed20_12 dram_efficiency; /* 0.7 */
2045 fixed20_12 yclk, dram_channels, bandwidth;
2046 fixed20_12 a;
2047
2048 a.full = dfixed_const(1000);
2049 yclk.full = dfixed_const(wm->yclk);
2050 yclk.full = dfixed_div(yclk, a);
2051 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2052 a.full = dfixed_const(10);
2053 dram_efficiency.full = dfixed_const(7);
2054 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2055 bandwidth.full = dfixed_mul(dram_channels, yclk);
2056 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2057
2058 return dfixed_trunc(bandwidth);
2059}
2060
2061static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2062{
2063 /* Calculate DRAM Bandwidth and the part allocated to display. */
2064 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2065 fixed20_12 yclk, dram_channels, bandwidth;
2066 fixed20_12 a;
2067
2068 a.full = dfixed_const(1000);
2069 yclk.full = dfixed_const(wm->yclk);
2070 yclk.full = dfixed_div(yclk, a);
2071 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2072 a.full = dfixed_const(10);
2073 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2074 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2075 bandwidth.full = dfixed_mul(dram_channels, yclk);
2076 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2077
2078 return dfixed_trunc(bandwidth);
2079}
2080
2081static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2082{
2083 /* Calculate the display Data return Bandwidth */
2084 fixed20_12 return_efficiency; /* 0.8 */
2085 fixed20_12 sclk, bandwidth;
2086 fixed20_12 a;
2087
2088 a.full = dfixed_const(1000);
2089 sclk.full = dfixed_const(wm->sclk);
2090 sclk.full = dfixed_div(sclk, a);
2091 a.full = dfixed_const(10);
2092 return_efficiency.full = dfixed_const(8);
2093 return_efficiency.full = dfixed_div(return_efficiency, a);
2094 a.full = dfixed_const(32);
2095 bandwidth.full = dfixed_mul(a, sclk);
2096 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2097
2098 return dfixed_trunc(bandwidth);
2099}
2100
2101static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2102{
2103 /* Calculate the DMIF Request Bandwidth */
2104 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2105 fixed20_12 disp_clk, bandwidth;
2106 fixed20_12 a;
2107
2108 a.full = dfixed_const(1000);
2109 disp_clk.full = dfixed_const(wm->disp_clk);
2110 disp_clk.full = dfixed_div(disp_clk, a);
2111 a.full = dfixed_const(10);
2112 disp_clk_request_efficiency.full = dfixed_const(8);
2113 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2114 a.full = dfixed_const(32);
2115 bandwidth.full = dfixed_mul(a, disp_clk);
2116 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2117
2118 return dfixed_trunc(bandwidth);
2119}
2120
2121static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2122{
2123 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2124 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2125 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2126 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2127
2128 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2129}
2130
2131static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2132{
2133 /* Calculate the display mode Average Bandwidth
2134 * DisplayMode should contain the source and destination dimensions,
2135 * timing, etc.
2136 */
2137 fixed20_12 bpp;
2138 fixed20_12 line_time;
2139 fixed20_12 src_width;
2140 fixed20_12 bandwidth;
2141 fixed20_12 a;
2142
2143 a.full = dfixed_const(1000);
2144 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2145 line_time.full = dfixed_div(line_time, a);
2146 bpp.full = dfixed_const(wm->bytes_per_pixel);
2147 src_width.full = dfixed_const(wm->src_width);
2148 bandwidth.full = dfixed_mul(src_width, bpp);
2149 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2150 bandwidth.full = dfixed_div(bandwidth, line_time);
2151
2152 return dfixed_trunc(bandwidth);
2153}
2154
2155static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2156{
2157 /* First calcualte the latency in ns */
2158 u32 mc_latency = 2000; /* 2000 ns. */
2159 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2160 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2161 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2162 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2163 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2164 (wm->num_heads * cursor_line_pair_return_time);
2165 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2166 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2167 fixed20_12 a, b, c;
2168
2169 if (wm->num_heads == 0)
2170 return 0;
2171
2172 a.full = dfixed_const(2);
2173 b.full = dfixed_const(1);
2174 if ((wm->vsc.full > a.full) ||
2175 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2176 (wm->vtaps >= 5) ||
2177 ((wm->vsc.full >= a.full) && wm->interlaced))
2178 max_src_lines_per_dst_line = 4;
2179 else
2180 max_src_lines_per_dst_line = 2;
2181
2182 a.full = dfixed_const(available_bandwidth);
2183 b.full = dfixed_const(wm->num_heads);
2184 a.full = dfixed_div(a, b);
2185
2186 b.full = dfixed_const(1000);
2187 c.full = dfixed_const(wm->disp_clk);
2188 b.full = dfixed_div(c, b);
2189 c.full = dfixed_const(wm->bytes_per_pixel);
2190 b.full = dfixed_mul(b, c);
2191
2192 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2193
2194 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2195 b.full = dfixed_const(1000);
2196 c.full = dfixed_const(lb_fill_bw);
2197 b.full = dfixed_div(c, b);
2198 a.full = dfixed_div(a, b);
2199 line_fill_time = dfixed_trunc(a);
2200
2201 if (line_fill_time < wm->active_time)
2202 return latency;
2203 else
2204 return latency + (line_fill_time - wm->active_time);
2205
2206}
2207
2208static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2209{
2210 if (evergreen_average_bandwidth(wm) <=
2211 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2212 return true;
2213 else
2214 return false;
2215};
2216
2217static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2218{
2219 if (evergreen_average_bandwidth(wm) <=
2220 (evergreen_available_bandwidth(wm) / wm->num_heads))
2221 return true;
2222 else
2223 return false;
2224};
2225
2226static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2227{
2228 u32 lb_partitions = wm->lb_size / wm->src_width;
2229 u32 line_time = wm->active_time + wm->blank_time;
2230 u32 latency_tolerant_lines;
2231 u32 latency_hiding;
2232 fixed20_12 a;
2233
2234 a.full = dfixed_const(1);
2235 if (wm->vsc.full > a.full)
2236 latency_tolerant_lines = 1;
2237 else {
2238 if (lb_partitions <= (wm->vtaps + 1))
2239 latency_tolerant_lines = 1;
2240 else
2241 latency_tolerant_lines = 2;
2242 }
2243
2244 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2245
2246 if (evergreen_latency_watermark(wm) <= latency_hiding)
2247 return true;
2248 else
2249 return false;
2250}
2251
2252static void evergreen_program_watermarks(struct radeon_device *rdev,
2253 struct radeon_crtc *radeon_crtc,
2254 u32 lb_size, u32 num_heads)
2255{
2256 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2257 struct evergreen_wm_params wm_low, wm_high;
2258 u32 dram_channels;
f9d9c362
AD		 2259 u32 pixel_period;
2260 u32 line_time = 0;
2261 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2262 u32 priority_a_mark = 0, priority_b_mark = 0;
2263 u32 priority_a_cnt = PRIORITY_OFF;
2264 u32 priority_b_cnt = PRIORITY_OFF;
2265 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2266 u32 tmp, arb_control3;
2267 fixed20_12 a, b, c;
2268
2269 if (radeon_crtc->base.enabled && num_heads && mode) {
2270 pixel_period = 1000000 / (u32)mode->clock;
2271 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2272 priority_a_cnt = 0;
2273 priority_b_cnt = 0;
cf0cfdd7
AD		 2274 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2275
2276 /* watermark for high clocks */
2277 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2278 wm_high.yclk =
2279 radeon_dpm_get_mclk(rdev, false) * 10;
2280 wm_high.sclk =
2281 radeon_dpm_get_sclk(rdev, false) * 10;
2282 } else {
2283 wm_high.yclk = rdev->pm.current_mclk * 10;
2284 wm_high.sclk = rdev->pm.current_sclk * 10;
2285 }
f9d9c362 2286
cf0cfdd7
AD		 2287 wm_high.disp_clk = mode->clock;
2288 wm_high.src_width = mode->crtc_hdisplay;
2289 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2290 wm_high.blank_time = line_time - wm_high.active_time;
2291 wm_high.interlaced = false;
f9d9c362 2292 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2293 wm_high.interlaced = true;
2294 wm_high.vsc = radeon_crtc->vsc;
2295 wm_high.vtaps = 1;
f9d9c362 2296 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2297 wm_high.vtaps = 2;
2298 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2299 wm_high.lb_size = lb_size;
2300 wm_high.dram_channels = dram_channels;
2301 wm_high.num_heads = num_heads;
2302
2303 /* watermark for low clocks */
2304 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2305 wm_low.yclk =
2306 radeon_dpm_get_mclk(rdev, true) * 10;
2307 wm_low.sclk =
2308 radeon_dpm_get_sclk(rdev, true) * 10;
2309 } else {
2310 wm_low.yclk = rdev->pm.current_mclk * 10;
2311 wm_low.sclk = rdev->pm.current_sclk * 10;
2312 }
2313
2314 wm_low.disp_clk = mode->clock;
2315 wm_low.src_width = mode->crtc_hdisplay;
2316 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2317 wm_low.blank_time = line_time - wm_low.active_time;
2318 wm_low.interlaced = false;
2319 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2320 wm_low.interlaced = true;
2321 wm_low.vsc = radeon_crtc->vsc;
2322 wm_low.vtaps = 1;
2323 if (radeon_crtc->rmx_type != RMX_OFF)
2324 wm_low.vtaps = 2;
2325 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2326 wm_low.lb_size = lb_size;
2327 wm_low.dram_channels = dram_channels;
2328 wm_low.num_heads = num_heads;
f9d9c362
AD		 2329
2330 /* set for high clocks */
cf0cfdd7 2331 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2332 /* set for low clocks */
cf0cfdd7 2333 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2334
2335 /* possibly force display priority to high */
2336 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2337 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2338 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2339 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2340 (rdev->disp_priority == 2)) {
cf0cfdd7 2341 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2342 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2343 }
2344 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2345 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2346 !evergreen_check_latency_hiding(&wm_low) ||
2347 (rdev->disp_priority == 2)) {
2348 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2349 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2350 }
2351
2352 a.full = dfixed_const(1000);
2353 b.full = dfixed_const(mode->clock);
2354 b.full = dfixed_div(b, a);
2355 c.full = dfixed_const(latency_watermark_a);
2356 c.full = dfixed_mul(c, b);
2357 c.full = dfixed_mul(c, radeon_crtc->hsc);
2358 c.full = dfixed_div(c, a);
2359 a.full = dfixed_const(16);
2360 c.full = dfixed_div(c, a);
2361 priority_a_mark = dfixed_trunc(c);
2362 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2363
2364 a.full = dfixed_const(1000);
2365 b.full = dfixed_const(mode->clock);
2366 b.full = dfixed_div(b, a);
2367 c.full = dfixed_const(latency_watermark_b);
2368 c.full = dfixed_mul(c, b);
2369 c.full = dfixed_mul(c, radeon_crtc->hsc);
2370 c.full = dfixed_div(c, a);
2371 a.full = dfixed_const(16);
2372 c.full = dfixed_div(c, a);
2373 priority_b_mark = dfixed_trunc(c);
2374 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2375 }
2376
2377 /* select wm A */
2378 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2379 tmp = arb_control3;
2380 tmp &= ~LATENCY_WATERMARK_MASK(3);
2381 tmp |= LATENCY_WATERMARK_MASK(1);
2382 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2383 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2384 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2385 LATENCY_HIGH_WATERMARK(line_time)));
2386 /* select wm B */
2387 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2388 tmp &= ~LATENCY_WATERMARK_MASK(3);
2389 tmp |= LATENCY_WATERMARK_MASK(2);
2390 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2391 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2392 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2393 LATENCY_HIGH_WATERMARK(line_time)));
2394 /* restore original selection */
2395 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2396
2397 /* write the priority marks */
2398 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2399 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2400
7178d2a6
AD		 2401 /* save values for DPM */
2402 radeon_crtc->line_time = line_time;
2403 radeon_crtc->wm_high = latency_watermark_a;
2404 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2405}
2406
377edc8b
AD		 2407/**
2408 * evergreen_bandwidth_update - update display watermarks callback.
2409 *
2410 * @rdev: radeon_device pointer
2411 *
2412 * Update the display watermarks based on the requested mode(s)
2413 * (evergreen+).
2414 */
0ca2ab52 2415void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2416{
f9d9c362
AD		 2417 struct drm_display_mode *mode0 = NULL;
2418 struct drm_display_mode *mode1 = NULL;
2419 u32 num_heads = 0, lb_size;
2420 int i;
2421
8efe82ca
AD		 2422 if (!rdev->mode_info.mode_config_initialized)
2423 return;
2424
f9d9c362
AD		 2425 radeon_update_display_priority(rdev);
2426
2427 for (i = 0; i < rdev->num_crtc; i++) {
2428 if (rdev->mode_info.crtcs[i]->base.enabled)
2429 num_heads++;
2430 }
2431 for (i = 0; i < rdev->num_crtc; i += 2) {
2432 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2433 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2434 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2435 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2436 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2437 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2438 }
bcc1c2a1
AD		 2439}
2440
377edc8b
AD		 2441/**
2442 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2443 *
2444 * @rdev: radeon_device pointer
2445 *
2446 * Wait for the MC (memory controller) to be idle.
2447 * (evergreen+).
2448 * Returns 0 if the MC is idle, -1 if not.
2449 */
b9952a8a 2450int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2451{
2452 unsigned i;
2453 u32 tmp;
2454
2455 for (i = 0; i < rdev->usec_timeout; i++) {
2456 /* read MC_STATUS */
2457 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2458 if (!tmp)
2459 return 0;
2460 udelay(1);
2461 }
2462 return -1;
2463}
2464
2465/*
2466 * GART
2467 */
0fcdb61e
AD		 2468void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2469{
2470 unsigned i;
2471 u32 tmp;
2472
6f2f48a9
AD		 2473 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2474
0fcdb61e
AD		 2475 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2476 for (i = 0; i < rdev->usec_timeout; i++) {
2477 /* read MC_STATUS */
2478 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2479 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2480 if (tmp == 2) {
2481 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2482 return;
2483 }
2484 if (tmp) {
2485 return;
2486 }
2487 udelay(1);
2488 }
2489}
2490
1109ca09 2491static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2492{
2493 u32 tmp;
0fcdb61e 2494 int r;
bcc1c2a1 2495
c9a1be96 2496 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2497 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2498 return -EINVAL;
2499 }
2500 r = radeon_gart_table_vram_pin(rdev);
2501 if (r)
2502 return r;
2503 /* Setup L2 cache */
2504 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2505 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2506 EFFECTIVE_L2_QUEUE_SIZE(7));
2507 WREG32(VM_L2_CNTL2, 0);
2508 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2509 /* Setup TLB control */
2510 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2511 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2512 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2513 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2514 if (rdev->flags & RADEON_IS_IGP) {
2515 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2516 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2517 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2518 } else {
2519 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2520 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2521 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2522 if ((rdev->family == CHIP_JUNIPER) ||
2523 (rdev->family == CHIP_CYPRESS) ||
2524 (rdev->family == CHIP_HEMLOCK) ||
2525 (rdev->family == CHIP_BARTS))
2526 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2527 }
bcc1c2a1
AD		 2528 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2529 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2530 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2531 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2532 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
7c0411d2 2533 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
bcc1c2a1
AD		 2534 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2535 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2536 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2537 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2538 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2539 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2540
0fcdb61e 2541 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2542 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2543 (unsigned)(rdev->mc.gtt_size >> 20),
2544 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2545 rdev->gart.ready = true;
2546 return 0;
2547}
2548
1109ca09 2549static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2550{
2551 u32 tmp;
bcc1c2a1
AD		 2552
2553 /* Disable all tables */
0fcdb61e
AD		 2554 WREG32(VM_CONTEXT0_CNTL, 0);
2555 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2556
2557 /* Setup L2 cache */
2558 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2559 EFFECTIVE_L2_QUEUE_SIZE(7));
2560 WREG32(VM_L2_CNTL2, 0);
2561 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2562 /* Setup TLB control */
2563 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2564 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2565 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2566 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2567 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2568 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2569 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2570 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2571 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2572}
2573
1109ca09 2574static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2575{
2576 evergreen_pcie_gart_disable(rdev);
2577 radeon_gart_table_vram_free(rdev);
2578 radeon_gart_fini(rdev);
2579}
2580
2581
1109ca09 2582static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2583{
2584 u32 tmp;
bcc1c2a1
AD		 2585
2586 /* Setup L2 cache */
2587 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2588 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2589 EFFECTIVE_L2_QUEUE_SIZE(7));
2590 WREG32(VM_L2_CNTL2, 0);
2591 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2592 /* Setup TLB control */
2593 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2594 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2595 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2596 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2597 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2598 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2599 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2600 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2601 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2602 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2603 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2604 WREG32(VM_CONTEXT0_CNTL, 0);
2605 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2606}
2607
b9952a8a 2608void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2609{
62444b74
AD		 2610 u32 crtc_enabled, tmp, frame_count, blackout;
2611 int i, j;
2612
5153550a
AD		 2613 if (!ASIC_IS_NODCE(rdev)) {
2614 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2615 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2616
5153550a
AD		 2617 /* disable VGA render */
2618 WREG32(VGA_RENDER_CONTROL, 0);
2619 }
62444b74
AD		 2620 /* blank the display controllers */
2621 for (i = 0; i < rdev->num_crtc; i++) {
2622 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2623 if (crtc_enabled) {
2624 save->crtc_enabled[i] = true;
2625 if (ASIC_IS_DCE6(rdev)) {
2626 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2627 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2628 radeon_wait_for_vblank(rdev, i);
abf1457b 2629 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2630 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2631 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
f0d7bfb9 2632 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2633 }
2634 } else {
2635 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2636 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2637 radeon_wait_for_vblank(rdev, i);
abf1457b 2638 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2639 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2640 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2641 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2642 }
2643 }
2644 /* wait for the next frame */
2645 frame_count = radeon_get_vblank_counter(rdev, i);
2646 for (j = 0; j < rdev->usec_timeout; j++) {
2647 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2648 break;
2649 udelay(1);
2650 }
abf1457b
AD		 2651
2652 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2653 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2654 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2655 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2656 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2657 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2658 save->crtc_enabled[i] = false;
2659 /* ***** */
804cc4a0
AD		 2660 } else {
2661 save->crtc_enabled[i] = false;
62444b74 2662 }
18007401 2663 }
bcc1c2a1 2664
62444b74
AD		 2665 radeon_mc_wait_for_idle(rdev);
2666
2667 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2668 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2669 /* Block CPU access */
2670 WREG32(BIF_FB_EN, 0);
2671 /* blackout the MC */
2672 blackout &= ~BLACKOUT_MODE_MASK;
2673 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2674 }
ed39fadd
AD		 2675 /* wait for the MC to settle */
2676 udelay(100);
968c0166
AD		 2677
2678 /* lock double buffered regs */
2679 for (i = 0; i < rdev->num_crtc; i++) {
2680 if (save->crtc_enabled[i]) {
2681 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2682 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2683 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2684 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2685 }
2686 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2687 if (!(tmp & 1)) {
2688 tmp |= 1;
2689 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2690 }
2691 }
2692 }
bcc1c2a1
AD		 2693}
2694
b9952a8a 2695void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2696{
62444b74
AD		 2697 u32 tmp, frame_count;
2698 int i, j;
18007401 2699
62444b74
AD		 2700 /* update crtc base addresses */
2701 for (i = 0; i < rdev->num_crtc; i++) {
2702 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2703 upper_32_bits(rdev->mc.vram_start));
62444b74 2704 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2705 upper_32_bits(rdev->mc.vram_start));
62444b74 2706 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2707 (u32)rdev->mc.vram_start);
62444b74 2708 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2709 (u32)rdev->mc.vram_start);
2710 }
5153550a
AD		 2711
2712 if (!ASIC_IS_NODCE(rdev)) {
2713 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2714 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2715 }
62444b74 2716
968c0166
AD		 2717 /* unlock regs and wait for update */
2718 for (i = 0; i < rdev->num_crtc; i++) {
2719 if (save->crtc_enabled[i]) {
2720 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
f53f81b2
MK		 2721 if ((tmp & 0x7) != 3) {
2722 tmp &= ~0x7;
2723 tmp |= 0x3;
968c0166
AD		 2724 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2725 }
2726 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2727 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2728 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2729 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2730 }
2731 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2732 if (tmp & 1) {
2733 tmp &= ~1;
2734 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2735 }
2736 for (j = 0; j < rdev->usec_timeout; j++) {
2737 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2738 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2739 break;
2740 udelay(1);
2741 }
2742 }
2743 }
2744
62444b74
AD		 2745 /* unblackout the MC */
2746 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2747 tmp &= ~BLACKOUT_MODE_MASK;
2748 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2749 /* allow CPU access */
2750 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2751
2752 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2753 if (save->crtc_enabled[i]) {
62444b74
AD		 2754 if (ASIC_IS_DCE6(rdev)) {
2755 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
3157c589 2756 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2757 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2758 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2759 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2760 } else {
2761 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2762 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2763 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2764 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2765 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2766 }
2767 /* wait for the next frame */
2768 frame_count = radeon_get_vblank_counter(rdev, i);
2769 for (j = 0; j < rdev->usec_timeout; j++) {
2770 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2771 break;
2772 udelay(1);
2773 }
2774 }
2775 }
5153550a
AD		 2776 if (!ASIC_IS_NODCE(rdev)) {
2777 /* Unlock vga access */
2778 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2779 mdelay(1);
2780 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2781 }
bcc1c2a1
AD		 2782}
2783
755d819e 2784void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2785{
2786 struct evergreen_mc_save save;
2787 u32 tmp;
2788 int i, j;
2789
2790 /* Initialize HDP */
2791 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2792 WREG32((0x2c14 + j), 0x00000000);
2793 WREG32((0x2c18 + j), 0x00000000);
2794 WREG32((0x2c1c + j), 0x00000000);
2795 WREG32((0x2c20 + j), 0x00000000);
2796 WREG32((0x2c24 + j), 0x00000000);
2797 }
2798 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2799
2800 evergreen_mc_stop(rdev, &save);
2801 if (evergreen_mc_wait_for_idle(rdev)) {
2802 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2803 }
2804 /* Lockout access through VGA aperture*/
2805 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2806 /* Update configuration */
2807 if (rdev->flags & RADEON_IS_AGP) {
2808 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2809 /* VRAM before AGP */
2810 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2811 rdev->mc.vram_start >> 12);
2812 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2813 rdev->mc.gtt_end >> 12);
2814 } else {
2815 /* VRAM after AGP */
2816 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2817 rdev->mc.gtt_start >> 12);
2818 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2819 rdev->mc.vram_end >> 12);
2820 }
2821 } else {
2822 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2823 rdev->mc.vram_start >> 12);
2824 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2825 rdev->mc.vram_end >> 12);
2826 }
3b9832f6 2827 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2828 /* llano/ontario only */
2829 if ((rdev->family == CHIP_PALM) ||
2830 (rdev->family == CHIP_SUMO) ||
2831 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2832 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2833 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2834 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2835 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2836 }
bcc1c2a1
AD		 2837 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2838 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2839 WREG32(MC_VM_FB_LOCATION, tmp);
2840 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2841 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2842 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2843 if (rdev->flags & RADEON_IS_AGP) {
2844 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2845 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2846 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2847 } else {
2848 WREG32(MC_VM_AGP_BASE, 0);
2849 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2850 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2851 }
2852 if (evergreen_mc_wait_for_idle(rdev)) {
2853 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2854 }
2855 evergreen_mc_resume(rdev, &save);
2856 /* we need to own VRAM, so turn off the VGA renderer here
2857 * to stop it overwriting our objects */
2858 rv515_vga_render_disable(rdev);
2859}
2860
bcc1c2a1
AD		 2861/*
2862 * CP.
2863 */
12920591
AD		 2864void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2865{
876dc9f3 2866 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2867 u32 next_rptr;
7b1f2485 2868
12920591 2869 /* set to DX10/11 mode */
e32eb50d
CK		 2870 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2871 radeon_ring_write(ring, 1);
45df6803
CK		 2872
2873 if (ring->rptr_save_reg) {
89d35807 2874 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2875 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2876 radeon_ring_write(ring, ((ring->rptr_save_reg -
2877 PACKET3_SET_CONFIG_REG_START) >> 2));
2878 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2879 } else if (rdev->wb.enabled) {
2880 next_rptr = ring->wptr + 5 + 4;
2881 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2882 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2883 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2884 radeon_ring_write(ring, next_rptr);
2885 radeon_ring_write(ring, 0);
45df6803
CK		 2886 }
2887
e32eb50d
CK		 2888 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2889 radeon_ring_write(ring,
0f234f5f
AD		 2890#ifdef __BIG_ENDIAN
2891 (2 << 0) |
2892#endif
2893 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2894 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2895 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2896}
2897
bcc1c2a1
AD		 2898
2899static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2900{
fe251e2f
AD		 2901 const __be32 *fw_data;
2902 int i;
2903
2904 if (!rdev->me_fw || !rdev->pfp_fw)
2905 return -EINVAL;
bcc1c2a1 2906
fe251e2f 2907 r700_cp_stop(rdev);
0f234f5f
AD		 2908 WREG32(CP_RB_CNTL,
2909#ifdef __BIG_ENDIAN
2910 BUF_SWAP_32BIT |
2911#endif
2912 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2913
2914 fw_data = (const __be32 *)rdev->pfp_fw->data;
2915 WREG32(CP_PFP_UCODE_ADDR, 0);
2916 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2917 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2918 WREG32(CP_PFP_UCODE_ADDR, 0);
2919
2920 fw_data = (const __be32 *)rdev->me_fw->data;
2921 WREG32(CP_ME_RAM_WADDR, 0);
2922 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2923 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2924
2925 WREG32(CP_PFP_UCODE_ADDR, 0);
2926 WREG32(CP_ME_RAM_WADDR, 0);
2927 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2928 return 0;
2929}
2930
7e7b41d2
AD		 2931static int evergreen_cp_start(struct radeon_device *rdev)
2932{
e32eb50d 2933 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2934 int r, i;
7e7b41d2
AD		 2935 uint32_t cp_me;
2936
e32eb50d 2937 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2938 if (r) {
2939 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2940 return r;
2941 }
e32eb50d
CK		 2942 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2943 radeon_ring_write(ring, 0x1);
2944 radeon_ring_write(ring, 0x0);
2945 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2946 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2947 radeon_ring_write(ring, 0);
2948 radeon_ring_write(ring, 0);
1538a9e0 2949 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 2950
2951 cp_me = 0xff;
2952 WREG32(CP_ME_CNTL, cp_me);
2953
e32eb50d 2954 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2955 if (r) {
2956 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2957 return r;
2958 }
2281a378
AD		 2959
2960 /* setup clear context state */
e32eb50d
CK		 2961 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2962 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2963
2964 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2965 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2966
e32eb50d
CK		 2967 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2968 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2969
2970 /* set clear context state */
e32eb50d
CK		 2971 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2972 radeon_ring_write(ring, 0);
2281a378
AD		 2973
2974 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2975 radeon_ring_write(ring, 0xc0026f00);
2976 radeon_ring_write(ring, 0x00000000);
2977 radeon_ring_write(ring, 0x00000000);
2978 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2979
2980 /* Clear consts */
e32eb50d
CK		 2981 radeon_ring_write(ring, 0xc0036f00);
2982 radeon_ring_write(ring, 0x00000bc4);
2983 radeon_ring_write(ring, 0xffffffff);
2984 radeon_ring_write(ring, 0xffffffff);
2985 radeon_ring_write(ring, 0xffffffff);
2281a378 2986
e32eb50d
CK		 2987 radeon_ring_write(ring, 0xc0026900);
2988 radeon_ring_write(ring, 0x00000316);
2989 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2990 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2991
1538a9e0 2992 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 2993
2994 return 0;
2995}
2996
1109ca09 2997static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2998{
e32eb50d 2999 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 3000 u32 tmp;
3001 u32 rb_bufsz;
3002 int r;
3003
3004 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3005 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3006 SOFT_RESET_PA |
3007 SOFT_RESET_SH |
3008 SOFT_RESET_VGT |
a49a50da 3009 SOFT_RESET_SPI |
fe251e2f
AD		 3010 SOFT_RESET_SX));
3011 RREG32(GRBM_SOFT_RESET);
3012 mdelay(15);
3013 WREG32(GRBM_SOFT_RESET, 0);
3014 RREG32(GRBM_SOFT_RESET);
3015
3016 /* Set ring buffer size */
b72a8925
DV		 3017 rb_bufsz = order_base_2(ring->ring_size / 8);
3018 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 3019#ifdef __BIG_ENDIAN
3020 tmp |= BUF_SWAP_32BIT;
32fcdbf4 3021#endif
fe251e2f 3022 WREG32(CP_RB_CNTL, tmp);
15d3332f 3023 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 3024 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 3025
3026 /* Set the write pointer delay */
3027 WREG32(CP_RB_WPTR_DELAY, 0);
3028
3029 /* Initialize the ring buffer's read and write pointers */
3030 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3031 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 3032 ring->wptr = 0;
3033 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 3034
48fc7f7e 3035 /* set the wb address whether it's enabled or not */
0f234f5f 3036 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 3037 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 3038 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3039 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3040
3041 if (rdev->wb.enabled)
3042 WREG32(SCRATCH_UMSK, 0xff);
3043 else {
3044 tmp |= RB_NO_UPDATE;
3045 WREG32(SCRATCH_UMSK, 0);
3046 }
3047
fe251e2f
AD		 3048 mdelay(1);
3049 WREG32(CP_RB_CNTL, tmp);
3050
e32eb50d 3051 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 3052 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3053
7e7b41d2 3054 evergreen_cp_start(rdev);
e32eb50d 3055 ring->ready = true;
f712812e 3056 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 3057 if (r) {
e32eb50d 3058 ring->ready = false;
fe251e2f
AD		 3059 return r;
3060 }
3061 return 0;
3062}
bcc1c2a1
AD		 3063
3064/*
3065 * Core functions
3066 */
bcc1c2a1
AD		 3067static void evergreen_gpu_init(struct radeon_device *rdev)
3068{
416a2bd2 3069 u32 gb_addr_config;
32fcdbf4 3070 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 3071 u32 sx_debug_1;
3072 u32 smx_dc_ctl0;
3073 u32 sq_config;
3074 u32 sq_lds_resource_mgmt;
3075 u32 sq_gpr_resource_mgmt_1;
3076 u32 sq_gpr_resource_mgmt_2;
3077 u32 sq_gpr_resource_mgmt_3;
3078 u32 sq_thread_resource_mgmt;
3079 u32 sq_thread_resource_mgmt_2;
3080 u32 sq_stack_resource_mgmt_1;
3081 u32 sq_stack_resource_mgmt_2;
3082 u32 sq_stack_resource_mgmt_3;
3083 u32 vgt_cache_invalidation;
f25a5c63 3084 u32 hdp_host_path_cntl, tmp;
416a2bd2 3085 u32 disabled_rb_mask;
072c44bf 3086 int i, j, ps_thread_count;
32fcdbf4
AD		 3087
3088 switch (rdev->family) {
3089 case CHIP_CYPRESS:
3090 case CHIP_HEMLOCK:
3091 rdev->config.evergreen.num_ses = 2;
3092 rdev->config.evergreen.max_pipes = 4;
3093 rdev->config.evergreen.max_tile_pipes = 8;
3094 rdev->config.evergreen.max_simds = 10;
3095 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3096 rdev->config.evergreen.max_gprs = 256;
3097 rdev->config.evergreen.max_threads = 248;
3098 rdev->config.evergreen.max_gs_threads = 32;
3099 rdev->config.evergreen.max_stack_entries = 512;
3100 rdev->config.evergreen.sx_num_of_sets = 4;
3101 rdev->config.evergreen.sx_max_export_size = 256;
3102 rdev->config.evergreen.sx_max_export_pos_size = 64;
3103 rdev->config.evergreen.sx_max_export_smx_size = 192;
3104 rdev->config.evergreen.max_hw_contexts = 8;
3105 rdev->config.evergreen.sq_num_cf_insts = 2;
3106
3107 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3108 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3109 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3110 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3111 break;
3112 case CHIP_JUNIPER:
3113 rdev->config.evergreen.num_ses = 1;
3114 rdev->config.evergreen.max_pipes = 4;
3115 rdev->config.evergreen.max_tile_pipes = 4;
3116 rdev->config.evergreen.max_simds = 10;
3117 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3118 rdev->config.evergreen.max_gprs = 256;
3119 rdev->config.evergreen.max_threads = 248;
3120 rdev->config.evergreen.max_gs_threads = 32;
3121 rdev->config.evergreen.max_stack_entries = 512;
3122 rdev->config.evergreen.sx_num_of_sets = 4;
3123 rdev->config.evergreen.sx_max_export_size = 256;
3124 rdev->config.evergreen.sx_max_export_pos_size = 64;
3125 rdev->config.evergreen.sx_max_export_smx_size = 192;
3126 rdev->config.evergreen.max_hw_contexts = 8;
3127 rdev->config.evergreen.sq_num_cf_insts = 2;
3128
3129 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3130 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3131 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3132 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3133 break;
3134 case CHIP_REDWOOD:
3135 rdev->config.evergreen.num_ses = 1;
3136 rdev->config.evergreen.max_pipes = 4;
3137 rdev->config.evergreen.max_tile_pipes = 4;
3138 rdev->config.evergreen.max_simds = 5;
3139 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3140 rdev->config.evergreen.max_gprs = 256;
3141 rdev->config.evergreen.max_threads = 248;
3142 rdev->config.evergreen.max_gs_threads = 32;
3143 rdev->config.evergreen.max_stack_entries = 256;
3144 rdev->config.evergreen.sx_num_of_sets = 4;
3145 rdev->config.evergreen.sx_max_export_size = 256;
3146 rdev->config.evergreen.sx_max_export_pos_size = 64;
3147 rdev->config.evergreen.sx_max_export_smx_size = 192;
3148 rdev->config.evergreen.max_hw_contexts = 8;
3149 rdev->config.evergreen.sq_num_cf_insts = 2;
3150
3151 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3152 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3153 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3154 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3155 break;
3156 case CHIP_CEDAR:
3157 default:
3158 rdev->config.evergreen.num_ses = 1;
3159 rdev->config.evergreen.max_pipes = 2;
3160 rdev->config.evergreen.max_tile_pipes = 2;
3161 rdev->config.evergreen.max_simds = 2;
3162 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3163 rdev->config.evergreen.max_gprs = 256;
3164 rdev->config.evergreen.max_threads = 192;
3165 rdev->config.evergreen.max_gs_threads = 16;
3166 rdev->config.evergreen.max_stack_entries = 256;
3167 rdev->config.evergreen.sx_num_of_sets = 4;
3168 rdev->config.evergreen.sx_max_export_size = 128;
3169 rdev->config.evergreen.sx_max_export_pos_size = 32;
3170 rdev->config.evergreen.sx_max_export_smx_size = 96;
3171 rdev->config.evergreen.max_hw_contexts = 4;
3172 rdev->config.evergreen.sq_num_cf_insts = 1;
3173
d5e455e4
AD		 3174 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3175 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3176 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3177 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3178 break;
3179 case CHIP_PALM:
3180 rdev->config.evergreen.num_ses = 1;
3181 rdev->config.evergreen.max_pipes = 2;
3182 rdev->config.evergreen.max_tile_pipes = 2;
3183 rdev->config.evergreen.max_simds = 2;
3184 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3185 rdev->config.evergreen.max_gprs = 256;
3186 rdev->config.evergreen.max_threads = 192;
3187 rdev->config.evergreen.max_gs_threads = 16;
3188 rdev->config.evergreen.max_stack_entries = 256;
3189 rdev->config.evergreen.sx_num_of_sets = 4;
3190 rdev->config.evergreen.sx_max_export_size = 128;
3191 rdev->config.evergreen.sx_max_export_pos_size = 32;
3192 rdev->config.evergreen.sx_max_export_smx_size = 96;
3193 rdev->config.evergreen.max_hw_contexts = 4;
3194 rdev->config.evergreen.sq_num_cf_insts = 1;
3195
d5c5a72f
AD		 3196 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3197 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3198 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3199 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3200 break;
3201 case CHIP_SUMO:
3202 rdev->config.evergreen.num_ses = 1;
3203 rdev->config.evergreen.max_pipes = 4;
bd25f078 3204 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3205 if (rdev->pdev->device == 0x9648)
3206 rdev->config.evergreen.max_simds = 3;
3207 else if ((rdev->pdev->device == 0x9647) ||
3208 (rdev->pdev->device == 0x964a))
3209 rdev->config.evergreen.max_simds = 4;
3210 else
3211 rdev->config.evergreen.max_simds = 5;
3212 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3213 rdev->config.evergreen.max_gprs = 256;
3214 rdev->config.evergreen.max_threads = 248;
3215 rdev->config.evergreen.max_gs_threads = 32;
3216 rdev->config.evergreen.max_stack_entries = 256;
3217 rdev->config.evergreen.sx_num_of_sets = 4;
3218 rdev->config.evergreen.sx_max_export_size = 256;
3219 rdev->config.evergreen.sx_max_export_pos_size = 64;
3220 rdev->config.evergreen.sx_max_export_smx_size = 192;
3221 rdev->config.evergreen.max_hw_contexts = 8;
3222 rdev->config.evergreen.sq_num_cf_insts = 2;
3223
3224 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3225 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3226 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3227 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3228 break;
3229 case CHIP_SUMO2:
3230 rdev->config.evergreen.num_ses = 1;
3231 rdev->config.evergreen.max_pipes = 4;
3232 rdev->config.evergreen.max_tile_pipes = 4;
3233 rdev->config.evergreen.max_simds = 2;
3234 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3235 rdev->config.evergreen.max_gprs = 256;
3236 rdev->config.evergreen.max_threads = 248;
3237 rdev->config.evergreen.max_gs_threads = 32;
3238 rdev->config.evergreen.max_stack_entries = 512;
3239 rdev->config.evergreen.sx_num_of_sets = 4;
3240 rdev->config.evergreen.sx_max_export_size = 256;
3241 rdev->config.evergreen.sx_max_export_pos_size = 64;
3242 rdev->config.evergreen.sx_max_export_smx_size = 192;
50b8f5ae 3243 rdev->config.evergreen.max_hw_contexts = 4;
d5c5a72f
AD		 3244 rdev->config.evergreen.sq_num_cf_insts = 2;
3245
adb68fa2
AD		 3246 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3247 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3248 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3249 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3250 break;
3251 case CHIP_BARTS:
3252 rdev->config.evergreen.num_ses = 2;
3253 rdev->config.evergreen.max_pipes = 4;
3254 rdev->config.evergreen.max_tile_pipes = 8;
3255 rdev->config.evergreen.max_simds = 7;
3256 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3257 rdev->config.evergreen.max_gprs = 256;
3258 rdev->config.evergreen.max_threads = 248;
3259 rdev->config.evergreen.max_gs_threads = 32;
3260 rdev->config.evergreen.max_stack_entries = 512;
3261 rdev->config.evergreen.sx_num_of_sets = 4;
3262 rdev->config.evergreen.sx_max_export_size = 256;
3263 rdev->config.evergreen.sx_max_export_pos_size = 64;
3264 rdev->config.evergreen.sx_max_export_smx_size = 192;
3265 rdev->config.evergreen.max_hw_contexts = 8;
3266 rdev->config.evergreen.sq_num_cf_insts = 2;
3267
3268 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3269 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3270 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3271 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3272 break;
3273 case CHIP_TURKS:
3274 rdev->config.evergreen.num_ses = 1;
3275 rdev->config.evergreen.max_pipes = 4;
3276 rdev->config.evergreen.max_tile_pipes = 4;
3277 rdev->config.evergreen.max_simds = 6;
3278 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3279 rdev->config.evergreen.max_gprs = 256;
3280 rdev->config.evergreen.max_threads = 248;
3281 rdev->config.evergreen.max_gs_threads = 32;
3282 rdev->config.evergreen.max_stack_entries = 256;
3283 rdev->config.evergreen.sx_num_of_sets = 4;
3284 rdev->config.evergreen.sx_max_export_size = 256;
3285 rdev->config.evergreen.sx_max_export_pos_size = 64;
3286 rdev->config.evergreen.sx_max_export_smx_size = 192;
3287 rdev->config.evergreen.max_hw_contexts = 8;
3288 rdev->config.evergreen.sq_num_cf_insts = 2;
3289
3290 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3291 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3292 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3293 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3294 break;
3295 case CHIP_CAICOS:
3296 rdev->config.evergreen.num_ses = 1;
bd25f078 3297 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3298 rdev->config.evergreen.max_tile_pipes = 2;
3299 rdev->config.evergreen.max_simds = 2;
3300 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3301 rdev->config.evergreen.max_gprs = 256;
3302 rdev->config.evergreen.max_threads = 192;
3303 rdev->config.evergreen.max_gs_threads = 16;
3304 rdev->config.evergreen.max_stack_entries = 256;
3305 rdev->config.evergreen.sx_num_of_sets = 4;
3306 rdev->config.evergreen.sx_max_export_size = 128;
3307 rdev->config.evergreen.sx_max_export_pos_size = 32;
3308 rdev->config.evergreen.sx_max_export_smx_size = 96;
3309 rdev->config.evergreen.max_hw_contexts = 4;
3310 rdev->config.evergreen.sq_num_cf_insts = 1;
3311
32fcdbf4
AD		 3312 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3313 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3314 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3315 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3316 break;
3317 }
3318
3319 /* Initialize HDP */
3320 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3321 WREG32((0x2c14 + j), 0x00000000);
3322 WREG32((0x2c18 + j), 0x00000000);
3323 WREG32((0x2c1c + j), 0x00000000);
3324 WREG32((0x2c20 + j), 0x00000000);
3325 WREG32((0x2c24 + j), 0x00000000);
3326 }
3327
3328 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
acc1522a
CK		 3329 WREG32(SRBM_INT_CNTL, 0x1);
3330 WREG32(SRBM_INT_ACK, 0x1);
32fcdbf4 3331
d054ac16
AD		 3332 evergreen_fix_pci_max_read_req_size(rdev);
3333
32fcdbf4 3334 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3335 if ((rdev->family == CHIP_PALM) ||
3336 (rdev->family == CHIP_SUMO) ||
3337 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3338 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3339 else
3340 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3341
1aa52bd3
AD		 3342 /* setup tiling info dword. gb_addr_config is not adequate since it does
3343 * not have bank info, so create a custom tiling dword.
3344 * bits 3:0 num_pipes
3345 * bits 7:4 num_banks
3346 * bits 11:8 group_size
3347 * bits 15:12 row_size
3348 */
3349 rdev->config.evergreen.tile_config = 0;
3350 switch (rdev->config.evergreen.max_tile_pipes) {
3351 case 1:
3352 default:
3353 rdev->config.evergreen.tile_config |= (0 << 0);
3354 break;
3355 case 2:
3356 rdev->config.evergreen.tile_config |= (1 << 0);
3357 break;
3358 case 4:
3359 rdev->config.evergreen.tile_config |= (2 << 0);
3360 break;
3361 case 8:
3362 rdev->config.evergreen.tile_config |= (3 << 0);
3363 break;
3364 }
d698a34d 3365 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3366 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3367 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3368 else {
c8d15edc
AD		 3369 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3370 case 0: /* four banks */
29d65406 3371 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3372 break;
3373 case 1: /* eight banks */
3374 rdev->config.evergreen.tile_config |= 1 << 4;
3375 break;
3376 case 2: /* sixteen banks */
3377 default:
3378 rdev->config.evergreen.tile_config |= 2 << 4;
3379 break;
3380 }
29d65406 3381 }
416a2bd2 3382 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3383 rdev->config.evergreen.tile_config |=
3384 ((gb_addr_config & 0x30000000) >> 28) << 12;
3385
416a2bd2
AD		 3386 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3387 u32 efuse_straps_4;
3388 u32 efuse_straps_3;
32fcdbf4 3389
ff82bbc4
AD		 3390 efuse_straps_4 = RREG32_RCU(0x204);
3391 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3392 tmp = (((efuse_straps_4 & 0xf) << 4) |
3393 ((efuse_straps_3 & 0xf0000000) >> 28));
3394 } else {
3395 tmp = 0;
3396 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3397 u32 rb_disable_bitmap;
3398
3399 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3400 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3401 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3402 tmp <<= 4;
3403 tmp |= rb_disable_bitmap;
32fcdbf4 3404 }
416a2bd2
AD		 3405 }
3406 /* enabled rb are just the one not disabled :) */
3407 disabled_rb_mask = tmp;
cedb655a
AD		 3408 tmp = 0;
3409 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3410 tmp |= (1 << i);
3411 /* if all the backends are disabled, fix it up here */
3412 if ((disabled_rb_mask & tmp) == tmp) {
3413 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3414 disabled_rb_mask &= ~(1 << i);
3415 }
32fcdbf4 3416
65fcf668
AD		 3417 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3418 u32 simd_disable_bitmap;
3419
3420 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3421 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3422 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3423 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3424 tmp <<= 16;
3425 tmp |= simd_disable_bitmap;
3426 }
3427 rdev->config.evergreen.active_simds = hweight32(~tmp);
3428
416a2bd2
AD		 3429 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3430 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3431
416a2bd2
AD		 3432 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3433 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3434 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3435 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3436 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3437 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3438 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3439
f7eb9730
AD		 3440 if ((rdev->config.evergreen.max_backends == 1) &&
3441 (rdev->flags & RADEON_IS_IGP)) {
3442 if ((disabled_rb_mask & 3) == 1) {
3443 /* RB0 disabled, RB1 enabled */
3444 tmp = 0x11111111;
3445 } else {
3446 /* RB1 disabled, RB0 enabled */
3447 tmp = 0x00000000;
3448 }
3449 } else {
3450 tmp = gb_addr_config & NUM_PIPES_MASK;
3451 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3452 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3453 }
416a2bd2 3454 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3455
3456 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3457 WREG32(CGTS_TCC_DISABLE, 0);
3458 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3459 WREG32(CGTS_USER_TCC_DISABLE, 0);
3460
3461 /* set HW defaults for 3D engine */
3462 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3463 ROQ_IB2_START(0x2b)));
3464
3465 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3466
3467 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3468 SYNC_GRADIENT |
3469 SYNC_WALKER |
3470 SYNC_ALIGNER));
3471
3472 sx_debug_1 = RREG32(SX_DEBUG_1);
3473 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3474 WREG32(SX_DEBUG_1, sx_debug_1);
3475
3476
3477 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3478 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3479 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3480 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3481
b866d133
AD		 3482 if (rdev->family <= CHIP_SUMO2)
3483 WREG32(SMX_SAR_CTL0, 0x00010000);
3484
32fcdbf4
AD		 3485 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3486 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3487 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3488
3489 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3490 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3491 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3492
3493 WREG32(VGT_NUM_INSTANCES, 1);
3494 WREG32(SPI_CONFIG_CNTL, 0);
3495 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3496 WREG32(CP_PERFMON_CNTL, 0);
3497
3498 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3499 FETCH_FIFO_HIWATER(0x4) |
3500 DONE_FIFO_HIWATER(0xe0) |
3501 ALU_UPDATE_FIFO_HIWATER(0x8)));
3502
3503 sq_config = RREG32(SQ_CONFIG);
3504 sq_config &= ~(PS_PRIO(3) |
3505 VS_PRIO(3) |
3506 GS_PRIO(3) |
3507 ES_PRIO(3));
3508 sq_config |= (VC_ENABLE |
3509 EXPORT_SRC_C |
3510 PS_PRIO(0) |
3511 VS_PRIO(1) |
3512 GS_PRIO(2) |
3513 ES_PRIO(3));
3514
d5e455e4
AD		 3515 switch (rdev->family) {
3516 case CHIP_CEDAR:
3517 case CHIP_PALM:
d5c5a72f
AD		 3518 case CHIP_SUMO:
3519 case CHIP_SUMO2:
adb68fa2 3520 case CHIP_CAICOS:
32fcdbf4
AD		 3521 /* no vertex cache */
3522 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3523 break;
3524 default:
3525 break;
3526 }
32fcdbf4
AD		 3527
3528 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3529
3530 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3531 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3532 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3533 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3534 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3535 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3536 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3537
d5e455e4
AD		 3538 switch (rdev->family) {
3539 case CHIP_CEDAR:
3540 case CHIP_PALM:
d5c5a72f
AD		 3541 case CHIP_SUMO:
3542 case CHIP_SUMO2:
32fcdbf4 3543 ps_thread_count = 96;
d5e455e4
AD		 3544 break;
3545 default:
32fcdbf4 3546 ps_thread_count = 128;
d5e455e4
AD		 3547 break;
3548 }
32fcdbf4
AD		 3549
3550 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3551 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3552 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3553 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3554 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3555 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3556
3557 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3558 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3559 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3560 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3561 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3562 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3563
3564 WREG32(SQ_CONFIG, sq_config);
3565 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3566 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3567 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3568 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3569 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3570 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3571 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3572 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3573 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3574 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3575
3576 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3577 FORCE_EOV_MAX_REZ_CNT(255)));
3578
d5e455e4
AD		 3579 switch (rdev->family) {
3580 case CHIP_CEDAR:
3581 case CHIP_PALM:
d5c5a72f
AD		 3582 case CHIP_SUMO:
3583 case CHIP_SUMO2:
adb68fa2 3584 case CHIP_CAICOS:
32fcdbf4 3585 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3586 break;
3587 default:
32fcdbf4 3588 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3589 break;
3590 }
32fcdbf4
AD		 3591 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3592 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3593
3594 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3595 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3596 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3597
60a4a3e0
AD		 3598 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3599 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3600
32fcdbf4
AD		 3601 WREG32(CB_PERF_CTR0_SEL_0, 0);
3602 WREG32(CB_PERF_CTR0_SEL_1, 0);
3603 WREG32(CB_PERF_CTR1_SEL_0, 0);
3604 WREG32(CB_PERF_CTR1_SEL_1, 0);
3605 WREG32(CB_PERF_CTR2_SEL_0, 0);
3606 WREG32(CB_PERF_CTR2_SEL_1, 0);
3607 WREG32(CB_PERF_CTR3_SEL_0, 0);
3608 WREG32(CB_PERF_CTR3_SEL_1, 0);
3609
60a4a3e0
AD		 3610 /* clear render buffer base addresses */
3611 WREG32(CB_COLOR0_BASE, 0);
3612 WREG32(CB_COLOR1_BASE, 0);
3613 WREG32(CB_COLOR2_BASE, 0);
3614 WREG32(CB_COLOR3_BASE, 0);
3615 WREG32(CB_COLOR4_BASE, 0);
3616 WREG32(CB_COLOR5_BASE, 0);
3617 WREG32(CB_COLOR6_BASE, 0);
3618 WREG32(CB_COLOR7_BASE, 0);
3619 WREG32(CB_COLOR8_BASE, 0);
3620 WREG32(CB_COLOR9_BASE, 0);
3621 WREG32(CB_COLOR10_BASE, 0);
3622 WREG32(CB_COLOR11_BASE, 0);
3623
3624 /* set the shader const cache sizes to 0 */
3625 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3626 WREG32(i, 0);
3627 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3628 WREG32(i, 0);
3629
f25a5c63
AD		 3630 tmp = RREG32(HDP_MISC_CNTL);
3631 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3632 WREG32(HDP_MISC_CNTL, tmp);
3633
32fcdbf4
AD		 3634 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3635 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3636
3637 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3638
3639 udelay(50);
3640
bcc1c2a1
AD		 3641}
3642
3643int evergreen_mc_init(struct radeon_device *rdev)
3644{
bcc1c2a1
AD		 3645 u32 tmp;
3646 int chansize, numchan;
bcc1c2a1
AD		 3647
3648 /* Get VRAM informations */
3649 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3650 if ((rdev->family == CHIP_PALM) ||
3651 (rdev->family == CHIP_SUMO) ||
3652 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3653 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3654 else
3655 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3656 if (tmp & CHANSIZE_OVERRIDE) {
3657 chansize = 16;
3658 } else if (tmp & CHANSIZE_MASK) {
3659 chansize = 64;
3660 } else {
3661 chansize = 32;
3662 }
3663 tmp = RREG32(MC_SHARED_CHMAP);
3664 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3665 case 0:
3666 default:
3667 numchan = 1;
3668 break;
3669 case 1:
3670 numchan = 2;
3671 break;
3672 case 2:
3673 numchan = 4;
3674 break;
3675 case 3:
3676 numchan = 8;
3677 break;
3678 }
3679 rdev->mc.vram_width = numchan * chansize;
3680 /* Could aper size report 0 ? */
01d73a69
JC		 3681 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3682 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3683 /* Setup GPU memory space */
05b3ef69
AD		 3684 if ((rdev->family == CHIP_PALM) ||
3685 (rdev->family == CHIP_SUMO) ||
3686 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3687 /* size in bytes on fusion */
3688 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3689 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3690 } else {
05b3ef69 3691 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3692 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3693 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3694 }
51e5fcd3 3695 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3696 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3697 radeon_update_bandwidth_info(rdev);
3698
bcc1c2a1
AD		 3699 return 0;
3700}
d594e46a 3701
187e3593 3702void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3703{
64c56e8c 3704 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3705 RREG32(GRBM_STATUS));
64c56e8c 3706 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3707 RREG32(GRBM_STATUS_SE0));
64c56e8c 3708 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3709 RREG32(GRBM_STATUS_SE1));
64c56e8c 3710 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3711 RREG32(SRBM_STATUS));
a65a4369
AD		 3712 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3713 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3714 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3715 RREG32(CP_STALLED_STAT1));
3716 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3717 RREG32(CP_STALLED_STAT2));
3718 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3719 RREG32(CP_BUSY_STAT));
3720 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3721 RREG32(CP_STAT));
eaaa6983
JG		 3722 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3723 RREG32(DMA_STATUS_REG));
168757ea
AD		 3724 if (rdev->family >= CHIP_CAYMAN) {
3725 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3726 RREG32(DMA_STATUS_REG + 0x800));
3727 }
0ecebb9e
AD		 3728}
3729
168757ea 3730bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3731{
a65a4369
AD		 3732 u32 crtc_hung = 0;
3733 u32 crtc_status[6];
3734 u32 i, j, tmp;
3735
3736 for (i = 0; i < rdev->num_crtc; i++) {
3737 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3738 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3739 crtc_hung |= (1 << i);
3740 }
3741 }
3742
3743 for (j = 0; j < 10; j++) {
3744 for (i = 0; i < rdev->num_crtc; i++) {
3745 if (crtc_hung & (1 << i)) {
3746 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3747 if (tmp != crtc_status[i])
3748 crtc_hung &= ~(1 << i);
3749 }
3750 }
3751 if (crtc_hung == 0)
3752 return false;
3753 udelay(100);
3754 }
3755
3756 return true;
3757}
3758
2483b4ea 3759u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
a65a4369
AD		 3760{
3761 u32 reset_mask = 0;
b7630473 3762 u32 tmp;
0ecebb9e 3763
a65a4369
AD		 3764 /* GRBM_STATUS */
3765 tmp = RREG32(GRBM_STATUS);
3766 if (tmp & (PA_BUSY | SC_BUSY |
3767 SH_BUSY | SX_BUSY |
3768 TA_BUSY | VGT_BUSY |
3769 DB_BUSY | CB_BUSY |
3770 SPI_BUSY | VGT_BUSY_NO_DMA))
3771 reset_mask |= RADEON_RESET_GFX;
3772
3773 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3774 CP_BUSY | CP_COHERENCY_BUSY))
3775 reset_mask |= RADEON_RESET_CP;
3776
3777 if (tmp & GRBM_EE_BUSY)
3778 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3779
a65a4369
AD		 3780 /* DMA_STATUS_REG */
3781 tmp = RREG32(DMA_STATUS_REG);
3782 if (!(tmp & DMA_IDLE))
3783 reset_mask |= RADEON_RESET_DMA;
3784
3785 /* SRBM_STATUS2 */
3786 tmp = RREG32(SRBM_STATUS2);
3787 if (tmp & DMA_BUSY)
3788 reset_mask |= RADEON_RESET_DMA;
3789
3790 /* SRBM_STATUS */
3791 tmp = RREG32(SRBM_STATUS);
3792 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3793 reset_mask |= RADEON_RESET_RLC;
3794
3795 if (tmp & IH_BUSY)
3796 reset_mask |= RADEON_RESET_IH;
3797
3798 if (tmp & SEM_BUSY)
3799 reset_mask |= RADEON_RESET_SEM;
3800
3801 if (tmp & GRBM_RQ_PENDING)
3802 reset_mask |= RADEON_RESET_GRBM;
3803
3804 if (tmp & VMC_BUSY)
3805 reset_mask |= RADEON_RESET_VMC;
3806
3807 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3808 MCC_BUSY | MCD_BUSY))
3809 reset_mask |= RADEON_RESET_MC;
3810
3811 if (evergreen_is_display_hung(rdev))
3812 reset_mask |= RADEON_RESET_DISPLAY;
3813
3814 /* VM_L2_STATUS */
3815 tmp = RREG32(VM_L2_STATUS);
3816 if (tmp & L2_BUSY)
3817 reset_mask |= RADEON_RESET_VMC;
3818
d808fc88
AD		 3819 /* Skip MC reset as it's mostly likely not hung, just busy */
3820 if (reset_mask & RADEON_RESET_MC) {
3821 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3822 reset_mask &= ~RADEON_RESET_MC;
3823 }
3824
a65a4369
AD		 3825 return reset_mask;
3826}
3827
3828static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3829{
3830 struct evergreen_mc_save save;
3831 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3832 u32 tmp;
19fc42ed 3833
0ecebb9e 3834 if (reset_mask == 0)
a65a4369 3835 return;
0ecebb9e
AD		 3836
3837 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3838
b7630473
AD		 3839 evergreen_print_gpu_status_regs(rdev);
3840
b7630473
AD		 3841 /* Disable CP parsing/prefetching */
3842 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3843
3844 if (reset_mask & RADEON_RESET_DMA) {
3845 /* Disable DMA */
3846 tmp = RREG32(DMA_RB_CNTL);
3847 tmp &= ~DMA_RB_ENABLE;
3848 WREG32(DMA_RB_CNTL, tmp);
3849 }
3850
b21b6e7a
AD		 3851 udelay(50);
3852
3853 evergreen_mc_stop(rdev, &save);
3854 if (evergreen_mc_wait_for_idle(rdev)) {
3855 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3856 }
3857
b7630473
AD		 3858 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3859 grbm_soft_reset |= SOFT_RESET_DB |
3860 SOFT_RESET_CB |
3861 SOFT_RESET_PA |
3862 SOFT_RESET_SC |
3863 SOFT_RESET_SPI |
3864 SOFT_RESET_SX |
3865 SOFT_RESET_SH |
3866 SOFT_RESET_TC |
3867 SOFT_RESET_TA |
3868 SOFT_RESET_VC |
3869 SOFT_RESET_VGT;
3870 }
3871
3872 if (reset_mask & RADEON_RESET_CP) {
3873 grbm_soft_reset |= SOFT_RESET_CP |
3874 SOFT_RESET_VGT;
3875
3876 srbm_soft_reset |= SOFT_RESET_GRBM;
3877 }
0ecebb9e
AD		 3878
3879 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3880 srbm_soft_reset |= SOFT_RESET_DMA;
3881
a65a4369
AD		 3882 if (reset_mask & RADEON_RESET_DISPLAY)
3883 srbm_soft_reset |= SOFT_RESET_DC;
3884
3885 if (reset_mask & RADEON_RESET_RLC)
3886 srbm_soft_reset |= SOFT_RESET_RLC;
3887
3888 if (reset_mask & RADEON_RESET_SEM)
3889 srbm_soft_reset |= SOFT_RESET_SEM;
3890
3891 if (reset_mask & RADEON_RESET_IH)
3892 srbm_soft_reset |= SOFT_RESET_IH;
3893
3894 if (reset_mask & RADEON_RESET_GRBM)
3895 srbm_soft_reset |= SOFT_RESET_GRBM;
3896
3897 if (reset_mask & RADEON_RESET_VMC)
3898 srbm_soft_reset |= SOFT_RESET_VMC;
3899
24178ec4
AD		 3900 if (!(rdev->flags & RADEON_IS_IGP)) {
3901 if (reset_mask & RADEON_RESET_MC)
3902 srbm_soft_reset |= SOFT_RESET_MC;
3903 }
a65a4369 3904
b7630473
AD		 3905 if (grbm_soft_reset) {
3906 tmp = RREG32(GRBM_SOFT_RESET);
3907 tmp |= grbm_soft_reset;
3908 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3909 WREG32(GRBM_SOFT_RESET, tmp);
3910 tmp = RREG32(GRBM_SOFT_RESET);
3911
3912 udelay(50);
3913
3914 tmp &= ~grbm_soft_reset;
3915 WREG32(GRBM_SOFT_RESET, tmp);
3916 tmp = RREG32(GRBM_SOFT_RESET);
3917 }
3918
3919 if (srbm_soft_reset) {
3920 tmp = RREG32(SRBM_SOFT_RESET);
3921 tmp |= srbm_soft_reset;
3922 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3923 WREG32(SRBM_SOFT_RESET, tmp);
3924 tmp = RREG32(SRBM_SOFT_RESET);
3925
3926 udelay(50);
3927
3928 tmp &= ~srbm_soft_reset;
3929 WREG32(SRBM_SOFT_RESET, tmp);
3930 tmp = RREG32(SRBM_SOFT_RESET);
3931 }
0ecebb9e
AD		 3932
3933 /* Wait a little for things to settle down */
3934 udelay(50);
3935
747943ea 3936 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3937 udelay(50);
3938
b7630473 3939 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3940}
3941
b5470b03
AD		 3942void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3943{
3944 struct evergreen_mc_save save;
3945 u32 tmp, i;
3946
3947 dev_info(rdev->dev, "GPU pci config reset\n");
3948
3949 /* disable dpm? */
3950
3951 /* Disable CP parsing/prefetching */
3952 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3953 udelay(50);
3954 /* Disable DMA */
3955 tmp = RREG32(DMA_RB_CNTL);
3956 tmp &= ~DMA_RB_ENABLE;
3957 WREG32(DMA_RB_CNTL, tmp);
3958 /* XXX other engines? */
3959
3960 /* halt the rlc */
3961 r600_rlc_stop(rdev);
3962
3963 udelay(50);
3964
3965 /* set mclk/sclk to bypass */
3966 rv770_set_clk_bypass_mode(rdev);
3967 /* disable BM */
3968 pci_clear_master(rdev->pdev);
3969 /* disable mem access */
3970 evergreen_mc_stop(rdev, &save);
3971 if (evergreen_mc_wait_for_idle(rdev)) {
3972 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3973 }
3974 /* reset */
3975 radeon_pci_config_reset(rdev);
3976 /* wait for asic to come out of reset */
3977 for (i = 0; i < rdev->usec_timeout; i++) {
3978 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3979 break;
3980 udelay(1);
3981 }
3982}
3983
a2d07b74 3984int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3985{
a65a4369
AD		 3986 u32 reset_mask;
3987
3988 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3989
3990 if (reset_mask)
3991 r600_set_bios_scratch_engine_hung(rdev, true);
3992
b5470b03 3993 /* try soft reset */
a65a4369
AD		 3994 evergreen_gpu_soft_reset(rdev, reset_mask);
3995
3996 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3997
b5470b03
AD		 3998 /* try pci config reset */
3999 if (reset_mask && radeon_hard_reset)
4000 evergreen_gpu_pci_config_reset(rdev);
4001
4002 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4003
a65a4369
AD		 4004 if (!reset_mask)
4005 r600_set_bios_scratch_engine_hung(rdev, false);
4006
4007 return 0;
747943ea
AD		 4008}
4009
123bc183
AD		 4010/**
4011 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4012 *
4013 * @rdev: radeon_device pointer
4014 * @ring: radeon_ring structure holding ring information
4015 *
4016 * Check if the GFX engine is locked up.
4017 * Returns true if the engine appears to be locked up, false if not.
4018 */
4019bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4020{
4021 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4022
4023 if (!(reset_mask & (RADEON_RESET_GFX |
4024 RADEON_RESET_COMPUTE |
4025 RADEON_RESET_CP))) {
ff212f25 4026 radeon_ring_lockup_update(rdev, ring);
123bc183
AD		 4027 return false;
4028 }
123bc183
AD		 4029 return radeon_ring_test_lockup(rdev, ring);
4030}
4031
2948f5e6
AD		 4032/*
4033 * RLC
4034 */
4035#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4036#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4037
4038void sumo_rlc_fini(struct radeon_device *rdev)
4039{
4040 int r;
4041
4042 /* save restore block */
4043 if (rdev->rlc.save_restore_obj) {
4044 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4045 if (unlikely(r != 0))
4046 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4047 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4048 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4049
4050 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4051 rdev->rlc.save_restore_obj = NULL;
4052 }
4053
4054 /* clear state block */
4055 if (rdev->rlc.clear_state_obj) {
4056 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4057 if (unlikely(r != 0))
4058 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4059 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4060 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4061
4062 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4063 rdev->rlc.clear_state_obj = NULL;
4064 }
22c775ce
AD		 4065
4066 /* clear state block */
4067 if (rdev->rlc.cp_table_obj) {
4068 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4069 if (unlikely(r != 0))
4070 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4071 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4072 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4073
4074 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4075 rdev->rlc.cp_table_obj = NULL;
4076 }
2948f5e6
AD		 4077}
4078
22c775ce
AD		 4079#define CP_ME_TABLE_SIZE 96
4080
2948f5e6
AD		 4081int sumo_rlc_init(struct radeon_device *rdev)
4082{
1fd11777 4083 const u32 *src_ptr;
2948f5e6
AD		 4084 volatile u32 *dst_ptr;
4085 u32 dws, data, i, j, k, reg_num;
59a82d0e 4086 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
2948f5e6 4087 u64 reg_list_mc_addr;
1fd11777 4088 const struct cs_section_def *cs_data;
2948f5e6
AD		 4089 int r;
4090
4091 src_ptr = rdev->rlc.reg_list;
4092 dws = rdev->rlc.reg_list_size;
a0f38609
AD		 4093 if (rdev->family >= CHIP_BONAIRE) {
4094 dws += (5 * 16) + 48 + 48 + 64;
4095 }
2948f5e6
AD		 4096 cs_data = rdev->rlc.cs_data;
4097
10b7ca7e
AD		 4098 if (src_ptr) {
4099 /* save restore block */
4100 if (rdev->rlc.save_restore_obj == NULL) {
4101 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4102 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4103 NULL, &rdev->rlc.save_restore_obj);
10b7ca7e
AD		 4104 if (r) {
4105 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4106 return r;
4107 }
4108 }
4109
4110 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4111 if (unlikely(r != 0)) {
4112 sumo_rlc_fini(rdev);
4113 return r;
4114 }
4115 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4116 &rdev->rlc.save_restore_gpu_addr);
2948f5e6 4117 if (r) {
10b7ca7e
AD		 4118 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4119 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4120 sumo_rlc_fini(rdev);
2948f5e6
AD		 4121 return r;
4122 }
2948f5e6 4123
10b7ca7e
AD		 4124 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4125 if (r) {
4126 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4127 sumo_rlc_fini(rdev);
4128 return r;
4129 }
4130 /* write the sr buffer */
4131 dst_ptr = rdev->rlc.sr_ptr;
1fd11777
AD		 4132 if (rdev->family >= CHIP_TAHITI) {
4133 /* SI */
59a82d0e 4134 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6ba81e53 4135 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
1fd11777
AD		 4136 } else {
4137 /* ON/LN/TN */
4138 /* format:
4139 * dw0: (reg2 << 16) | reg1
4140 * dw1: reg1 save space
4141 * dw2: reg2 save space
4142 */
4143 for (i = 0; i < dws; i++) {
4144 data = src_ptr[i] >> 2;
4145 i++;
4146 if (i < dws)
4147 data |= (src_ptr[i] >> 2) << 16;
4148 j = (((i - 1) * 3) / 2);
6ba81e53 4149 dst_ptr[j] = cpu_to_le32(data);
1fd11777
AD		 4150 }
4151 j = ((i * 3) / 2);
6ba81e53 4152 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
10b7ca7e 4153 }
10b7ca7e 4154 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
2948f5e6 4155 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2948f5e6 4156 }
2948f5e6 4157
10b7ca7e
AD		 4158 if (cs_data) {
4159 /* clear state block */
a0f38609
AD		 4160 if (rdev->family >= CHIP_BONAIRE) {
4161 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4162 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4163 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4164 dws = rdev->rlc.clear_state_size + (256 / 4);
4165 } else {
4166 reg_list_num = 0;
4167 dws = 0;
4168 for (i = 0; cs_data[i].section != NULL; i++) {
4169 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4170 reg_list_num++;
4171 dws += cs_data[i].section[j].reg_count;
4172 }
10b7ca7e 4173 }
59a82d0e
AD		 4174 reg_list_blk_index = (3 * reg_list_num + 2);
4175 dws += reg_list_blk_index;
4176 rdev->rlc.clear_state_size = dws;
2948f5e6 4177 }
2948f5e6 4178
10b7ca7e 4179 if (rdev->rlc.clear_state_obj == NULL) {
59a82d0e 4180 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4181 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4182 NULL, &rdev->rlc.clear_state_obj);
10b7ca7e
AD		 4183 if (r) {
4184 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4185 sumo_rlc_fini(rdev);
4186 return r;
4187 }
4188 }
4189 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4190 if (unlikely(r != 0)) {
4191 sumo_rlc_fini(rdev);
4192 return r;
4193 }
4194 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4195 &rdev->rlc.clear_state_gpu_addr);
2948f5e6 4196 if (r) {
10b7ca7e
AD		 4197 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4198 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
2948f5e6
AD		 4199 sumo_rlc_fini(rdev);
4200 return r;
4201 }
2948f5e6 4202
10b7ca7e
AD		 4203 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4204 if (r) {
4205 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4206 sumo_rlc_fini(rdev);
4207 return r;
4208 }
4209 /* set up the cs buffer */
4210 dst_ptr = rdev->rlc.cs_ptr;
a0f38609
AD		 4211 if (rdev->family >= CHIP_BONAIRE) {
4212 cik_get_csb_buffer(rdev, dst_ptr);
4213 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e 4214 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
6ba81e53
AD		 4215 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4216 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4217 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
59a82d0e
AD		 4218 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4219 } else {
4220 reg_list_hdr_blk_index = 0;
4221 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4222 data = upper_32_bits(reg_list_mc_addr);
6ba81e53 4223 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
2948f5e6 4224 reg_list_hdr_blk_index++;
59a82d0e
AD		 4225 for (i = 0; cs_data[i].section != NULL; i++) {
4226 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4227 reg_num = cs_data[i].section[j].reg_count;
4228 data = reg_list_mc_addr & 0xffffffff;
6ba81e53 4229 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4230 reg_list_hdr_blk_index++;
4231
4232 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
6ba81e53 4233 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4234 reg_list_hdr_blk_index++;
4235
4236 data = 0x08000000 | (reg_num * 4);
6ba81e53 4237 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4238 reg_list_hdr_blk_index++;
4239
4240 for (k = 0; k < reg_num; k++) {
4241 data = cs_data[i].section[j].extent[k];
6ba81e53 4242 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
59a82d0e
AD		 4243 }
4244 reg_list_mc_addr += reg_num * 4;
4245 reg_list_blk_index += reg_num;
10b7ca7e 4246 }
2948f5e6 4247 }
6ba81e53 4248 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
2948f5e6 4249 }
10b7ca7e
AD		 4250 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4251 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
2948f5e6 4252 }
2948f5e6 4253
22c775ce
AD		 4254 if (rdev->rlc.cp_table_size) {
4255 if (rdev->rlc.cp_table_obj == NULL) {
02376d82
MD		 4256 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4257 PAGE_SIZE, true,
4258 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4259 NULL, &rdev->rlc.cp_table_obj);
22c775ce
AD		 4260 if (r) {
4261 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4262 sumo_rlc_fini(rdev);
4263 return r;
4264 }
4265 }
4266
4267 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4268 if (unlikely(r != 0)) {
4269 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4270 sumo_rlc_fini(rdev);
4271 return r;
4272 }
4273 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4274 &rdev->rlc.cp_table_gpu_addr);
4275 if (r) {
4276 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4277 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4278 sumo_rlc_fini(rdev);
4279 return r;
4280 }
4281 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4282 if (r) {
4283 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4284 sumo_rlc_fini(rdev);
4285 return r;
4286 }
4287
4288 cik_init_cp_pg_table(rdev);
4289
4290 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4291 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4292
4293 }
2948f5e6
AD		 4294
4295 return 0;
4296}
4297
4298static void evergreen_rlc_start(struct radeon_device *rdev)
4299{
8ba10463
AD		 4300 u32 mask = RLC_ENABLE;
4301
4302 if (rdev->flags & RADEON_IS_IGP) {
4303 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
8ba10463
AD		 4304 }
4305
4306 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4307}
4308
4309int evergreen_rlc_resume(struct radeon_device *rdev)
4310{
4311 u32 i;
4312 const __be32 *fw_data;
4313
4314 if (!rdev->rlc_fw)
4315 return -EINVAL;
4316
4317 r600_rlc_stop(rdev);
4318
4319 WREG32(RLC_HB_CNTL, 0);
4320
4321 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4322 if (rdev->family == CHIP_ARUBA) {
4323 u32 always_on_bitmap =
4324 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4325 /* find out the number of active simds */
4326 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4327 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4328 tmp = hweight32(~tmp);
4329 if (tmp == rdev->config.cayman.max_simds_per_se) {
4330 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4331 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4332 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4333 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4334 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4335 }
4336 } else {
4337 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4338 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4339 }
2948f5e6
AD		 4340 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4341 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4342 } else {
4343 WREG32(RLC_HB_BASE, 0);
4344 WREG32(RLC_HB_RPTR, 0);
4345 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4346 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4347 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4348 }
2948f5e6
AD		 4349 WREG32(RLC_MC_CNTL, 0);
4350 WREG32(RLC_UCODE_CNTL, 0);
4351
4352 fw_data = (const __be32 *)rdev->rlc_fw->data;
4353 if (rdev->family >= CHIP_ARUBA) {
4354 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4355 WREG32(RLC_UCODE_ADDR, i);
4356 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4357 }
4358 } else if (rdev->family >= CHIP_CAYMAN) {
4359 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4360 WREG32(RLC_UCODE_ADDR, i);
4361 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4362 }
4363 } else {
4364 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4365 WREG32(RLC_UCODE_ADDR, i);
4366 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4367 }
4368 }
4369 WREG32(RLC_UCODE_ADDR, 0);
4370
4371 evergreen_rlc_start(rdev);
4372
4373 return 0;
4374}
4375
45f9a39b
AD		 4376/* Interrupts */
4377
4378u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4379{
46437057 4380 if (crtc >= rdev->num_crtc)
45f9a39b 4381 return 0;
46437057
AD		 4382 else
4383 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4384}
4385
4386void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4387{
4388 u32 tmp;
4389
1b37078b
AD		 4390 if (rdev->family >= CHIP_CAYMAN) {
4391 cayman_cp_int_cntl_setup(rdev, 0,
4392 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4393 cayman_cp_int_cntl_setup(rdev, 1, 0);
4394 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4395 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4396 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4397 } else
4398 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4399 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4400 WREG32(DMA_CNTL, tmp);
45f9a39b 4401 WREG32(GRBM_INT_CNTL, 0);
acc1522a 4402 WREG32(SRBM_INT_CNTL, 0);
45f9a39b
AD		 4403 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4404 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4405 if (rdev->num_crtc >= 4) {
18007401
AD		 4406 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4407 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4408 }
4409 if (rdev->num_crtc >= 6) {
18007401
AD		 4410 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4411 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4412 }
45f9a39b
AD		 4413
4414 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4415 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4416 if (rdev->num_crtc >= 4) {
18007401
AD		 4417 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4418 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4419 }
4420 if (rdev->num_crtc >= 6) {
18007401
AD		 4421 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4422 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4423 }
45f9a39b 4424
e9a321c6
AD		 4425 /* only one DAC on DCE5 */
4426 if (!ASIC_IS_DCE5(rdev))
05b3ef69 4427 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4428 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4429
4430 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4431 WREG32(DC_HPD1_INT_CONTROL, tmp);
4432 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4433 WREG32(DC_HPD2_INT_CONTROL, tmp);
4434 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4435 WREG32(DC_HPD3_INT_CONTROL, tmp);
4436 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4437 WREG32(DC_HPD4_INT_CONTROL, tmp);
4438 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4439 WREG32(DC_HPD5_INT_CONTROL, tmp);
4440 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4441 WREG32(DC_HPD6_INT_CONTROL, tmp);
4442
4443}
4444
4445int evergreen_irq_set(struct radeon_device *rdev)
4446{
4447 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4448 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4449 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4450 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4451 u32 grbm_int_cntl = 0;
f122c610 4452 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4453 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4454 u32 thermal_int = 0;
45f9a39b
AD		 4455
4456 if (!rdev->irq.installed) {
fce7d61b 4457 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4458 return -EINVAL;
4459 }
4460 /* don't enable anything if the ih is disabled */
4461 if (!rdev->ih.enabled) {
4462 r600_disable_interrupts(rdev);
4463 /* force the active interrupt state to all disabled */
4464 evergreen_disable_interrupt_state(rdev);
4465 return 0;
4466 }
4467
2bc67b4d
DA		 4468 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4469 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4470 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4471 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4472 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4473 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
d70229f7
AD		 4474 if (rdev->family == CHIP_ARUBA)
4475 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4476 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4477 else
4478 thermal_int = RREG32(CG_THERMAL_INT) &
4479 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4480
f122c610
AD		 4481 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4482 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4483 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4484 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4485 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4486 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4487
233d1ad5
AD		 4488 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4489
1b37078b
AD		 4490 if (rdev->family >= CHIP_CAYMAN) {
4491 /* enable CP interrupts on all rings */
736fc37f 4492 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4493 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4494 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4495 }
736fc37f 4496 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4497 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4498 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4499 }
736fc37f 4500 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4501 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4502 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4503 }
4504 } else {
736fc37f 4505 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4506 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4507 cp_int_cntl |= RB_INT_ENABLE;
4508 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4509 }
45f9a39b 4510 }
1b37078b 4511
233d1ad5
AD		 4512 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4513 DRM_DEBUG("r600_irq_set: sw int dma\n");
4514 dma_cntl |= TRAP_ENABLE;
4515 }
4516
f60cbd11
AD		 4517 if (rdev->family >= CHIP_CAYMAN) {
4518 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4519 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4520 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4521 dma_cntl1 |= TRAP_ENABLE;
4522 }
4523 }
4524
dc50ba7f
AD		 4525 if (rdev->irq.dpm_thermal) {
4526 DRM_DEBUG("dpm thermal\n");
4527 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4528 }
4529
6f34be50 4530 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4531 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4532 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4533 crtc1 |= VBLANK_INT_MASK;
4534 }
6f34be50 4535 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4536 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4537 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4538 crtc2 |= VBLANK_INT_MASK;
4539 }
6f34be50 4540 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4541 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4542 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4543 crtc3 |= VBLANK_INT_MASK;
4544 }
6f34be50 4545 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4546 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4547 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4548 crtc4 |= VBLANK_INT_MASK;
4549 }
6f34be50 4550 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4551 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4552 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4553 crtc5 |= VBLANK_INT_MASK;
4554 }
6f34be50 4555 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4556 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4557 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4558 crtc6 |= VBLANK_INT_MASK;
4559 }
4560 if (rdev->irq.hpd[0]) {
4561 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
2bc67b4d 4562 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4563 }
4564 if (rdev->irq.hpd[1]) {
4565 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
2bc67b4d 4566 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4567 }
4568 if (rdev->irq.hpd[2]) {
4569 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
2bc67b4d 4570 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4571 }
4572 if (rdev->irq.hpd[3]) {
4573 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
2bc67b4d 4574 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4575 }
4576 if (rdev->irq.hpd[4]) {
4577 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
2bc67b4d 4578 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4579 }
4580 if (rdev->irq.hpd[5]) {
4581 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
2bc67b4d 4582 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b 4583 }
f122c610
AD		 4584 if (rdev->irq.afmt[0]) {
4585 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4586 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4587 }
4588 if (rdev->irq.afmt[1]) {
4589 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4590 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4591 }
4592 if (rdev->irq.afmt[2]) {
4593 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4594 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4595 }
4596 if (rdev->irq.afmt[3]) {
4597 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4598 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4599 }
4600 if (rdev->irq.afmt[4]) {
4601 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4602 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4603 }
4604 if (rdev->irq.afmt[5]) {
4605 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4606 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4607 }
45f9a39b 4608
1b37078b
AD		 4609 if (rdev->family >= CHIP_CAYMAN) {
4610 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4611 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4612 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4613 } else
4614 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4615
4616 WREG32(DMA_CNTL, dma_cntl);
4617
f60cbd11
AD		 4618 if (rdev->family >= CHIP_CAYMAN)
4619 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4620
2031f77c 4621 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4622
4623 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4624 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4625 if (rdev->num_crtc >= 4) {
18007401
AD		 4626 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4627 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4628 }
4629 if (rdev->num_crtc >= 6) {
18007401
AD		 4630 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4631 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4632 }
45f9a39b 4633
f5d636d2
CK		 4634 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4635 GRPH_PFLIP_INT_MASK);
4636 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4637 GRPH_PFLIP_INT_MASK);
b7eff394 4638 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 4639 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4640 GRPH_PFLIP_INT_MASK);
4641 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4642 GRPH_PFLIP_INT_MASK);
b7eff394
AD		 4643 }
4644 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 4645 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4646 GRPH_PFLIP_INT_MASK);
4647 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4648 GRPH_PFLIP_INT_MASK);
b7eff394 4649 }
6f34be50 4650
45f9a39b
AD		 4651 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4652 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4653 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4654 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4655 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4656 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4657 if (rdev->family == CHIP_ARUBA)
4658 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4659 else
4660 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4661
f122c610
AD		 4662 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4663 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4664 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4665 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4666 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4667 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4668
c320bb5f
AD		 4669 /* posting read */
4670 RREG32(SRBM_STATUS);
4671
bcc1c2a1
AD		 4672 return 0;
4673}
4674
cbdd4501 4675static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4676{
4677 u32 tmp;
4678
6f34be50
AD		 4679 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4680 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4681 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4682 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4683 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4684 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4685 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4686 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4687 if (rdev->num_crtc >= 4) {
4688 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4689 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4690 }
4691 if (rdev->num_crtc >= 6) {
4692 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4693 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4694 }
6f34be50 4695
f122c610
AD		 4696 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4697 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4698 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4699 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4700 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4701 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4702
6f34be50
AD		 4703 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4704 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4705 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4706 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4707 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4708 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4709 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4710 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4711 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4712 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4713 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4714 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4715
b7eff394
AD		 4716 if (rdev->num_crtc >= 4) {
4717 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4718 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4719 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4720 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4721 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4722 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4723 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4724 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4725 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4726 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4727 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4728 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4729 }
4730
4731 if (rdev->num_crtc >= 6) {
4732 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4733 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4734 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4735 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4736 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4737 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4738 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4739 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4740 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4741 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4742 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4743 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4744 }
45f9a39b 4745
6f34be50 4746 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4747 tmp = RREG32(DC_HPD1_INT_CONTROL);
4748 tmp |= DC_HPDx_INT_ACK;
4749 WREG32(DC_HPD1_INT_CONTROL, tmp);
4750 }
6f34be50 4751 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4752 tmp = RREG32(DC_HPD2_INT_CONTROL);
4753 tmp |= DC_HPDx_INT_ACK;
4754 WREG32(DC_HPD2_INT_CONTROL, tmp);
4755 }
6f34be50 4756 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4757 tmp = RREG32(DC_HPD3_INT_CONTROL);
4758 tmp |= DC_HPDx_INT_ACK;
4759 WREG32(DC_HPD3_INT_CONTROL, tmp);
4760 }
6f34be50 4761 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4762 tmp = RREG32(DC_HPD4_INT_CONTROL);
4763 tmp |= DC_HPDx_INT_ACK;
4764 WREG32(DC_HPD4_INT_CONTROL, tmp);
4765 }
6f34be50 4766 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4767 tmp = RREG32(DC_HPD5_INT_CONTROL);
4768 tmp |= DC_HPDx_INT_ACK;
4769 WREG32(DC_HPD5_INT_CONTROL, tmp);
4770 }
6f34be50 4771 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4772 tmp = RREG32(DC_HPD5_INT_CONTROL);
4773 tmp |= DC_HPDx_INT_ACK;
4774 WREG32(DC_HPD6_INT_CONTROL, tmp);
4775 }
2bc67b4d
DA		 4776
4777 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
4778 tmp = RREG32(DC_HPD1_INT_CONTROL);
4779 tmp |= DC_HPDx_RX_INT_ACK;
4780 WREG32(DC_HPD1_INT_CONTROL, tmp);
4781 }
4782 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
4783 tmp = RREG32(DC_HPD2_INT_CONTROL);
4784 tmp |= DC_HPDx_RX_INT_ACK;
4785 WREG32(DC_HPD2_INT_CONTROL, tmp);
4786 }
4787 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
4788 tmp = RREG32(DC_HPD3_INT_CONTROL);
4789 tmp |= DC_HPDx_RX_INT_ACK;
4790 WREG32(DC_HPD3_INT_CONTROL, tmp);
4791 }
4792 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
4793 tmp = RREG32(DC_HPD4_INT_CONTROL);
4794 tmp |= DC_HPDx_RX_INT_ACK;
4795 WREG32(DC_HPD4_INT_CONTROL, tmp);
4796 }
4797 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
4798 tmp = RREG32(DC_HPD5_INT_CONTROL);
4799 tmp |= DC_HPDx_RX_INT_ACK;
4800 WREG32(DC_HPD5_INT_CONTROL, tmp);
4801 }
4802 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
4803 tmp = RREG32(DC_HPD5_INT_CONTROL);
4804 tmp |= DC_HPDx_RX_INT_ACK;
4805 WREG32(DC_HPD6_INT_CONTROL, tmp);
4806 }
4807
f122c610
AD		 4808 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4809 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4810 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4811 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4812 }
4813 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4814 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4815 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4816 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4817 }
4818 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4819 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4820 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4821 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4822 }
4823 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4824 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4825 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4826 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4827 }
4828 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4829 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4830 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4831 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4832 }
4833 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4834 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4835 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4836 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4837 }
45f9a39b
AD		 4838}
4839
1109ca09 4840static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4841{
45f9a39b
AD		 4842 r600_disable_interrupts(rdev);
4843 /* Wait and acknowledge irq */
4844 mdelay(1);
6f34be50 4845 evergreen_irq_ack(rdev);
45f9a39b
AD		 4846 evergreen_disable_interrupt_state(rdev);
4847}
4848
755d819e 4849void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4850{
4851 evergreen_irq_disable(rdev);
4852 r600_rlc_stop(rdev);
4853}
4854
cbdd4501 4855static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4856{
4857 u32 wptr, tmp;
4858
724c80e1 4859 if (rdev->wb.enabled)
204ae24d 4860 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4861 else
4862 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4863
4864 if (wptr & RB_OVERFLOW) {
11bab0ae 4865 wptr &= ~RB_OVERFLOW;
45f9a39b
AD		 4866 /* When a ring buffer overflow happen start parsing interrupt
4867 * from the last not overwritten vector (wptr + 16). Hopefully
4868 * this should allow us to catchup.
4869 */
6cc2fda2
MD		 4870 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4871 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
45f9a39b
AD		 4872 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4873 tmp = RREG32(IH_RB_CNTL);
4874 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4875 WREG32(IH_RB_CNTL, tmp);
4876 }
4877 return (wptr & rdev->ih.ptr_mask);
4878}
4879
4880int evergreen_irq_process(struct radeon_device *rdev)
4881{
682f1a54
DA		 4882 u32 wptr;
4883 u32 rptr;
45f9a39b
AD		 4884 u32 src_id, src_data;
4885 u32 ring_index;
45f9a39b 4886 bool queue_hotplug = false;
f122c610 4887 bool queue_hdmi = false;
2bc67b4d 4888 bool queue_dp = false;
dc50ba7f 4889 bool queue_thermal = false;
54e2e49c 4890 u32 status, addr;
45f9a39b 4891
682f1a54 4892 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4893 return IRQ_NONE;
4894
682f1a54 4895 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4896
4897restart_ih:
4898 /* is somebody else already processing irqs? */
4899 if (atomic_xchg(&rdev->ih.lock, 1))
4900 return IRQ_NONE;
4901
682f1a54 4902 rptr = rdev->ih.rptr;
07f18f0b 4903 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4904
964f6645
BH		 4905 /* Order reading of wptr vs. reading of IH ring data */
4906 rmb();
4907
45f9a39b 4908 /* display interrupts */
6f34be50 4909 evergreen_irq_ack(rdev);
45f9a39b 4910
45f9a39b
AD		 4911 while (rptr != wptr) {
4912 /* wptr/rptr are in bytes! */
4913 ring_index = rptr / 4;
0f234f5f
AD		 4914 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4915 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4916
4917 switch (src_id) {
4918 case 1: /* D1 vblank/vline */
4919 switch (src_data) {
4920 case 0: /* D1 vblank */
07f18f0b
MK		 4921 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
4922 DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n");
4923
4924 if (rdev->irq.crtc_vblank_int[0]) {
4925 drm_handle_vblank(rdev->ddev, 0);
4926 rdev->pm.vblank_sync = true;
4927 wake_up(&rdev->irq.vblank_queue);
45f9a39b 4928 }
07f18f0b
MK		 4929 if (atomic_read(&rdev->irq.pflip[0]))
4930 radeon_crtc_handle_vblank(rdev, 0);
4931 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4932 DRM_DEBUG("IH: D1 vblank\n");
4933
45f9a39b
AD		 4934 break;
4935 case 1: /* D1 vline */
07f18f0b
MK		 4936 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
4937 DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n");
4938
4939 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4940 DRM_DEBUG("IH: D1 vline\n");
4941
45f9a39b
AD		 4942 break;
4943 default:
4944 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4945 break;
4946 }
4947 break;
4948 case 2: /* D2 vblank/vline */
4949 switch (src_data) {
4950 case 0: /* D2 vblank */
07f18f0b
MK		 4951 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
4952 DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n");
4953
4954 if (rdev->irq.crtc_vblank_int[1]) {
4955 drm_handle_vblank(rdev->ddev, 1);
4956 rdev->pm.vblank_sync = true;
4957 wake_up(&rdev->irq.vblank_queue);
45f9a39b 4958 }
07f18f0b
MK		 4959 if (atomic_read(&rdev->irq.pflip[1]))
4960 radeon_crtc_handle_vblank(rdev, 1);
4961 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4962 DRM_DEBUG("IH: D2 vblank\n");
4963
45f9a39b
AD		 4964 break;
4965 case 1: /* D2 vline */
07f18f0b
MK		 4966 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
4967 DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n");
4968
4969 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4970 DRM_DEBUG("IH: D2 vline\n");
4971
45f9a39b
AD		 4972 break;
4973 default:
4974 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4975 break;
4976 }
4977 break;
4978 case 3: /* D3 vblank/vline */
4979 switch (src_data) {
4980 case 0: /* D3 vblank */
07f18f0b
MK		 4981 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
4982 DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n");
4983
4984 if (rdev->irq.crtc_vblank_int[2]) {
4985 drm_handle_vblank(rdev->ddev, 2);
4986 rdev->pm.vblank_sync = true;
4987 wake_up(&rdev->irq.vblank_queue);
45f9a39b 4988 }
07f18f0b
MK		 4989 if (atomic_read(&rdev->irq.pflip[2]))
4990 radeon_crtc_handle_vblank(rdev, 2);
4991 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4992 DRM_DEBUG("IH: D3 vblank\n");
4993
45f9a39b
AD		 4994 break;
4995 case 1: /* D3 vline */
07f18f0b
MK		 4996 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
4997 DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n");
4998
4999 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5000 DRM_DEBUG("IH: D3 vline\n");
5001
45f9a39b
AD		 5002 break;
5003 default:
5004 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5005 break;
5006 }
5007 break;
5008 case 4: /* D4 vblank/vline */
5009 switch (src_data) {
5010 case 0: /* D4 vblank */
07f18f0b
MK		 5011 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
5012 DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n");
5013
5014 if (rdev->irq.crtc_vblank_int[3]) {
5015 drm_handle_vblank(rdev->ddev, 3);
5016 rdev->pm.vblank_sync = true;
5017 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5018 }
07f18f0b
MK		 5019 if (atomic_read(&rdev->irq.pflip[3]))
5020 radeon_crtc_handle_vblank(rdev, 3);
5021 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5022 DRM_DEBUG("IH: D4 vblank\n");
5023
45f9a39b
AD		 5024 break;
5025 case 1: /* D4 vline */
07f18f0b
MK		 5026 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
5027 DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n");
5028
5029 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5030 DRM_DEBUG("IH: D4 vline\n");
5031
45f9a39b
AD		 5032 break;
5033 default:
5034 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5035 break;
5036 }
5037 break;
5038 case 5: /* D5 vblank/vline */
5039 switch (src_data) {
5040 case 0: /* D5 vblank */
07f18f0b
MK		 5041 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
5042 DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n");
5043
5044 if (rdev->irq.crtc_vblank_int[4]) {
5045 drm_handle_vblank(rdev->ddev, 4);
5046 rdev->pm.vblank_sync = true;
5047 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5048 }
07f18f0b
MK		 5049 if (atomic_read(&rdev->irq.pflip[4]))
5050 radeon_crtc_handle_vblank(rdev, 4);
5051 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5052 DRM_DEBUG("IH: D5 vblank\n");
5053
45f9a39b
AD		 5054 break;
5055 case 1: /* D5 vline */
07f18f0b
MK		 5056 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
5057 DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n");
5058
5059 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5060 DRM_DEBUG("IH: D5 vline\n");
5061
45f9a39b
AD		 5062 break;
5063 default:
5064 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5065 break;
5066 }
5067 break;
5068 case 6: /* D6 vblank/vline */
5069 switch (src_data) {
5070 case 0: /* D6 vblank */
07f18f0b
MK		 5071 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
5072 DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n");
5073
5074 if (rdev->irq.crtc_vblank_int[5]) {
5075 drm_handle_vblank(rdev->ddev, 5);
5076 rdev->pm.vblank_sync = true;
5077 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5078 }
07f18f0b
MK		 5079 if (atomic_read(&rdev->irq.pflip[5]))
5080 radeon_crtc_handle_vblank(rdev, 5);
5081 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5082 DRM_DEBUG("IH: D6 vblank\n");
5083
45f9a39b
AD		 5084 break;
5085 case 1: /* D6 vline */
07f18f0b
MK		 5086 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
5087 DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n");
5088
5089 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5090 DRM_DEBUG("IH: D6 vline\n");
5091
45f9a39b
AD		 5092 break;
5093 default:
5094 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5095 break;
5096 }
5097 break;
f5d636d2
CK		 5098 case 8: /* D1 page flip */
5099 case 10: /* D2 page flip */
5100 case 12: /* D3 page flip */
5101 case 14: /* D4 page flip */
5102 case 16: /* D5 page flip */
5103 case 18: /* D6 page flip */
5104 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 5105 if (radeon_use_pflipirq > 0)
5106 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 5107 break;
45f9a39b
AD		 5108 case 42: /* HPD hotplug */
5109 switch (src_data) {
5110 case 0:
07f18f0b
MK		 5111 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
5112 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5113
5114 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5115 queue_hotplug = true;
5116 DRM_DEBUG("IH: HPD1\n");
45f9a39b
AD		 5117 break;
5118 case 1:
07f18f0b
MK		 5119 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
5120 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5121
5122 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5123 queue_hotplug = true;
5124 DRM_DEBUG("IH: HPD2\n");
45f9a39b
AD		 5125 break;
5126 case 2:
07f18f0b
MK		 5127 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
5128 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5129
5130 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5131 queue_hotplug = true;
5132 DRM_DEBUG("IH: HPD3\n");
45f9a39b
AD		 5133 break;
5134 case 3:
07f18f0b
MK		 5135 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
5136 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5137
5138 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5139 queue_hotplug = true;
5140 DRM_DEBUG("IH: HPD4\n");
45f9a39b
AD		 5141 break;
5142 case 4:
07f18f0b
MK		 5143 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
5144 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5145
5146 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5147 queue_hotplug = true;
5148 DRM_DEBUG("IH: HPD5\n");
45f9a39b
AD		 5149 break;
5150 case 5:
07f18f0b
MK		 5151 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
5152 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5153
5154 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5155 queue_hotplug = true;
5156 DRM_DEBUG("IH: HPD6\n");
45f9a39b 5157 break;
2bc67b4d 5158 case 6:
07f18f0b
MK		 5159 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
5160 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5161
5162 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
5163 queue_dp = true;
5164 DRM_DEBUG("IH: HPD_RX 1\n");
2bc67b4d
DA		 5165 break;
5166 case 7:
07f18f0b
MK		 5167 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
5168 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5169
5170 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
5171 queue_dp = true;
5172 DRM_DEBUG("IH: HPD_RX 2\n");
2bc67b4d
DA		 5173 break;
5174 case 8:
07f18f0b
MK		 5175 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
5176 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5177
5178 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
5179 queue_dp = true;
5180 DRM_DEBUG("IH: HPD_RX 3\n");
2bc67b4d
DA		 5181 break;
5182 case 9:
07f18f0b
MK		 5183 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
5184 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5185
5186 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
5187 queue_dp = true;
5188 DRM_DEBUG("IH: HPD_RX 4\n");
2bc67b4d
DA		 5189 break;
5190 case 10:
07f18f0b
MK		 5191 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
5192 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5193
5194 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
5195 queue_dp = true;
5196 DRM_DEBUG("IH: HPD_RX 5\n");
2bc67b4d
DA		 5197 break;
5198 case 11:
07f18f0b
MK		 5199 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
5200 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5201
5202 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
5203 queue_dp = true;
5204 DRM_DEBUG("IH: HPD_RX 6\n");
2bc67b4d 5205 break;
45f9a39b
AD		 5206 default:
5207 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5208 break;
5209 }
5210 break;
f122c610
AD		 5211 case 44: /* hdmi */
5212 switch (src_data) {
5213 case 0:
07f18f0b
MK		 5214 if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG))
5215 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5216
5217 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5218 queue_hdmi = true;
5219 DRM_DEBUG("IH: HDMI0\n");
f122c610
AD		 5220 break;
5221 case 1:
07f18f0b
MK		 5222 if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG))
5223 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5224
5225 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5226 queue_hdmi = true;
5227 DRM_DEBUG("IH: HDMI1\n");
f122c610
AD		 5228 break;
5229 case 2:
07f18f0b
MK		 5230 if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG))
5231 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5232
5233 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5234 queue_hdmi = true;
5235 DRM_DEBUG("IH: HDMI2\n");
f122c610
AD		 5236 break;
5237 case 3:
07f18f0b
MK		 5238 if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG))
5239 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5240
5241 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5242 queue_hdmi = true;
5243 DRM_DEBUG("IH: HDMI3\n");
f122c610
AD		 5244 break;
5245 case 4:
07f18f0b
MK		 5246 if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG))
5247 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5248
5249 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5250 queue_hdmi = true;
5251 DRM_DEBUG("IH: HDMI4\n");
f122c610
AD		 5252 break;
5253 case 5:
07f18f0b
MK		 5254 if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG))
5255 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5256
5257 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5258 queue_hdmi = true;
5259 DRM_DEBUG("IH: HDMI5\n");
f122c610
AD		 5260 break;
5261 default:
5262 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5263 break;
5264 }
acc1522a
CK		 5265 case 96:
5266 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5267 WREG32(SRBM_INT_ACK, 0x1);
5268 break;
f2ba57b5
CK		 5269 case 124: /* UVD */
5270 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5271 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 5272 break;
ae133a11
CK		 5273 case 146:
5274 case 147:
54e2e49c
AD		 5275 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5276 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
9b7d786b
CK		 5277 /* reset addr and status */
5278 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5279 if (addr == 0x0 && status == 0x0)
5280 break;
ae133a11
CK		 5281 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5282 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
54e2e49c 5283 addr);
ae133a11 5284 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
54e2e49c
AD		 5285 status);
5286 cayman_vm_decode_fault(rdev, status, addr);
ae133a11 5287 break;
45f9a39b
AD		 5288 case 176: /* CP_INT in ring buffer */
5289 case 177: /* CP_INT in IB1 */
5290 case 178: /* CP_INT in IB2 */
5291 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 5292 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 5293 break;
5294 case 181: /* CP EOP event */
5295 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 5296 if (rdev->family >= CHIP_CAYMAN) {
5297 switch (src_data) {
5298 case 0:
5299 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5300 break;
5301 case 1:
5302 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5303 break;
5304 case 2:
5305 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5306 break;
5307 }
5308 } else
5309 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 5310 break;
233d1ad5
AD		 5311 case 224: /* DMA trap event */
5312 DRM_DEBUG("IH: DMA trap\n");
5313 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5314 break;
dc50ba7f
AD		 5315 case 230: /* thermal low to high */
5316 DRM_DEBUG("IH: thermal low to high\n");
5317 rdev->pm.dpm.thermal.high_to_low = false;
5318 queue_thermal = true;
5319 break;
5320 case 231: /* thermal high to low */
5321 DRM_DEBUG("IH: thermal high to low\n");
5322 rdev->pm.dpm.thermal.high_to_low = true;
5323 queue_thermal = true;
5324 break;
2031f77c 5325 case 233: /* GUI IDLE */
303c805c 5326 DRM_DEBUG("IH: GUI idle\n");
2031f77c 5327 break;
f60cbd11
AD		 5328 case 244: /* DMA trap event */
5329 if (rdev->family >= CHIP_CAYMAN) {
5330 DRM_DEBUG("IH: DMA1 trap\n");
5331 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5332 }
5333 break;
45f9a39b
AD		 5334 default:
5335 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5336 break;
5337 }
5338
5339 /* wptr/rptr are in bytes! */
5340 rptr += 16;
5341 rptr &= rdev->ih.ptr_mask;
f55e03b9 5342 WREG32(IH_RB_RPTR, rptr);
45f9a39b 5343 }
2bc67b4d
DA		 5344 if (queue_dp)
5345 schedule_work(&rdev->dp_work);
45f9a39b 5346 if (queue_hotplug)
cb5d4166 5347 schedule_delayed_work(&rdev->hotplug_work, 0);
f122c610
AD		 5348 if (queue_hdmi)
5349 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 5350 if (queue_thermal && rdev->pm.dpm_enabled)
5351 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b 5352 rdev->ih.rptr = rptr;
c20dc369
CK		 5353 atomic_set(&rdev->ih.lock, 0);
5354
5355 /* make sure wptr hasn't changed while processing */
5356 wptr = evergreen_get_ih_wptr(rdev);
5357 if (wptr != rptr)
5358 goto restart_ih;
5359
45f9a39b
AD		 5360 return IRQ_HANDLED;
5361}
5362
bcc1c2a1
AD		 5363static int evergreen_startup(struct radeon_device *rdev)
5364{
f2ba57b5 5365 struct radeon_ring *ring;
bcc1c2a1
AD		 5366 int r;
5367
9e46a48d 5368 /* enable pcie gen2 link */
cd54033a 5369 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5370 /* enable aspm */
5371 evergreen_program_aspm(rdev);
9e46a48d 5372
e5903d39
AD		 5373 /* scratch needs to be initialized before MC */
5374 r = r600_vram_scratch_init(rdev);
5375 if (r)
5376 return r;
5377
6fab3feb
AD		 5378 evergreen_mc_program(rdev);
5379
6c7bccea 5380 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
755d819e 5381 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5382 if (r) {
0af62b01 5383 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5384 return r;
5385 }
5386 }
fe251e2f 5387
bcc1c2a1 5388 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5389 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5390 } else {
5391 r = evergreen_pcie_gart_enable(rdev);
5392 if (r)
5393 return r;
5394 }
bcc1c2a1 5395 evergreen_gpu_init(rdev);
bcc1c2a1 5396
2948f5e6
AD		 5397 /* allocate rlc buffers */
5398 if (rdev->flags & RADEON_IS_IGP) {
5399 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
1fd11777
AD		 5400 rdev->rlc.reg_list_size =
5401 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
2948f5e6
AD		 5402 rdev->rlc.cs_data = evergreen_cs_data;
5403 r = sumo_rlc_init(rdev);
5404 if (r) {
5405 DRM_ERROR("Failed to init rlc BOs!\n");
5406 return r;
5407 }
5408 }
5409
724c80e1
AD		 5410 /* allocate wb buffer */
5411 r = radeon_wb_init(rdev);
5412 if (r)
5413 return r;
5414
30eb77f4
JG		 5415 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5416 if (r) {
5417 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5418 return r;
5419 }
5420
233d1ad5
AD		 5421 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5422 if (r) {
5423 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5424 return r;
5425 }
5426
e409b128 5427 r = uvd_v2_2_resume(rdev);
f2ba57b5
CK		 5428 if (!r) {
5429 r = radeon_fence_driver_start_ring(rdev,
5430 R600_RING_TYPE_UVD_INDEX);
5431 if (r)
5432 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5433 }
5434
5435 if (r)
5436 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5437
bcc1c2a1 5438 /* Enable IRQ */
e49f3959
AH		 5439 if (!rdev->irq.installed) {
5440 r = radeon_irq_kms_init(rdev);
5441 if (r)
5442 return r;
5443 }
5444
bcc1c2a1
AD		 5445 r = r600_irq_init(rdev);
5446 if (r) {
5447 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5448 radeon_irq_kms_fini(rdev);
5449 return r;
5450 }
45f9a39b 5451 evergreen_irq_set(rdev);
bcc1c2a1 5452
f2ba57b5 5453 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5454 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 5455 RADEON_CP_PACKET2);
bcc1c2a1
AD		 5456 if (r)
5457 return r;
233d1ad5
AD		 5458
5459 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5460 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 5461 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5462 if (r)
5463 return r;
5464
bcc1c2a1
AD		 5465 r = evergreen_cp_load_microcode(rdev);
5466 if (r)
5467 return r;
fe251e2f 5468 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5469 if (r)
5470 return r;
5471 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5472 if (r)
5473 return r;
fe251e2f 5474
f2ba57b5
CK		 5475 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5476 if (ring->ring_size) {
02c9f7fa 5477 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 5478 RADEON_CP_PACKET2);
f2ba57b5 5479 if (!r)
e409b128 5480 r = uvd_v1_0_init(rdev);
f2ba57b5
CK		 5481
5482 if (r)
5483 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5484 }
5485
2898c348
CK		 5486 r = radeon_ib_pool_init(rdev);
5487 if (r) {
5488 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5489 return r;
2898c348 5490 }
b15ba512 5491
bfc1f97d 5492 r = radeon_audio_init(rdev);
69d2ae57
RM		 5493 if (r) {
5494 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5495 return r;
5496 }
5497
bcc1c2a1
AD		 5498 return 0;
5499}
5500
5501int evergreen_resume(struct radeon_device *rdev)
5502{
5503 int r;
5504
86f5c9ed
AD		 5505 /* reset the asic, the gfx blocks are often in a bad state
5506 * after the driver is unloaded or after a resume
5507 */
5508 if (radeon_asic_reset(rdev))
5509 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5510 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5511 * posting will perform necessary task to bring back GPU into good
5512 * shape.
5513 */
5514 /* post card */
5515 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5516
d4788db3
AD		 5517 /* init golden registers */
5518 evergreen_init_golden_registers(rdev);
5519
bc6a6295
AD		 5520 if (rdev->pm.pm_method == PM_METHOD_DPM)
5521 radeon_pm_resume(rdev);
6c7bccea 5522
b15ba512 5523 rdev->accel_working = true;
bcc1c2a1
AD		 5524 r = evergreen_startup(rdev);
5525 if (r) {
755d819e 5526 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5527 rdev->accel_working = false;
bcc1c2a1
AD		 5528 return r;
5529 }
fe251e2f 5530
bcc1c2a1
AD		 5531 return r;
5532
5533}
5534
5535int evergreen_suspend(struct radeon_device *rdev)
5536{
6c7bccea 5537 radeon_pm_suspend(rdev);
7991d665 5538 radeon_audio_fini(rdev);
e409b128 5539 uvd_v1_0_fini(rdev);
f2ba57b5 5540 radeon_uvd_suspend(rdev);
bcc1c2a1 5541 r700_cp_stop(rdev);
233d1ad5 5542 r600_dma_stop(rdev);
45f9a39b 5543 evergreen_irq_suspend(rdev);
724c80e1 5544 radeon_wb_disable(rdev);
bcc1c2a1 5545 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5546
5547 return 0;
5548}
5549
bcc1c2a1
AD		 5550/* Plan is to move initialization in that function and use
5551 * helper function so that radeon_device_init pretty much
5552 * do nothing more than calling asic specific function. This
5553 * should also allow to remove a bunch of callback function
5554 * like vram_info.
5555 */
5556int evergreen_init(struct radeon_device *rdev)
5557{
5558 int r;
5559
bcc1c2a1
AD		 5560 /* Read BIOS */
5561 if (!radeon_get_bios(rdev)) {
5562 if (ASIC_IS_AVIVO(rdev))
5563 return -EINVAL;
5564 }
5565 /* Must be an ATOMBIOS */
5566 if (!rdev->is_atom_bios) {
755d819e 5567 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5568 return -EINVAL;
5569 }
5570 r = radeon_atombios_init(rdev);
5571 if (r)
5572 return r;
86f5c9ed
AD		 5573 /* reset the asic, the gfx blocks are often in a bad state
5574 * after the driver is unloaded or after a resume
5575 */
5576 if (radeon_asic_reset(rdev))
5577 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5578 /* Post card if necessary */
fd909c37 5579 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5580 if (!rdev->bios) {
5581 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5582 return -EINVAL;
5583 }
5584 DRM_INFO("GPU not posted. posting now...\n");
5585 atom_asic_init(rdev->mode_info.atom_context);
5586 }
d4788db3
AD		 5587 /* init golden registers */
5588 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5589 /* Initialize scratch registers */
5590 r600_scratch_init(rdev);
5591 /* Initialize surface registers */
5592 radeon_surface_init(rdev);
5593 /* Initialize clocks */
5594 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5595 /* Fence driver */
5596 r = radeon_fence_driver_init(rdev);
5597 if (r)
5598 return r;
d594e46a
JG		 5599 /* initialize AGP */
5600 if (rdev->flags & RADEON_IS_AGP) {
5601 r = radeon_agp_init(rdev);
5602 if (r)
5603 radeon_agp_disable(rdev);
5604 }
5605 /* initialize memory controller */
bcc1c2a1
AD		 5606 r = evergreen_mc_init(rdev);
5607 if (r)
5608 return r;
5609 /* Memory manager */
5610 r = radeon_bo_init(rdev);
5611 if (r)
5612 return r;
45f9a39b 5613
01ac8794
AD		 5614 if (ASIC_IS_DCE5(rdev)) {
5615 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5616 r = ni_init_microcode(rdev);
5617 if (r) {
5618 DRM_ERROR("Failed to load firmware!\n");
5619 return r;
5620 }
5621 }
5622 } else {
5623 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5624 r = r600_init_microcode(rdev);
5625 if (r) {
5626 DRM_ERROR("Failed to load firmware!\n");
5627 return r;
5628 }
5629 }
5630 }
5631
6c7bccea
AD		 5632 /* Initialize power management */
5633 radeon_pm_init(rdev);
5634
e32eb50d
CK		 5635 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5636 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5637
233d1ad5
AD		 5638 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5639 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5640
f2ba57b5
CK		 5641 r = radeon_uvd_init(rdev);
5642 if (!r) {
5643 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5644 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5645 4096);
5646 }
5647
bcc1c2a1
AD		 5648 rdev->ih.ring_obj = NULL;
5649 r600_ih_ring_init(rdev, 64 * 1024);
5650
5651 r = r600_pcie_gart_init(rdev);
5652 if (r)
5653 return r;
0fcdb61e 5654
148a03bc 5655 rdev->accel_working = true;
bcc1c2a1
AD		 5656 r = evergreen_startup(rdev);
5657 if (r) {
fe251e2f
AD		 5658 dev_err(rdev->dev, "disabling GPU acceleration\n");
5659 r700_cp_fini(rdev);
233d1ad5 5660 r600_dma_fini(rdev);
fe251e2f 5661 r600_irq_fini(rdev);
2948f5e6
AD		 5662 if (rdev->flags & RADEON_IS_IGP)
5663 sumo_rlc_fini(rdev);
724c80e1 5664 radeon_wb_fini(rdev);
2898c348 5665 radeon_ib_pool_fini(rdev);
fe251e2f 5666 radeon_irq_kms_fini(rdev);
0fcdb61e 5667 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5668 rdev->accel_working = false;
5669 }
77e00f2e
AD		 5670
5671 /* Don't start up if the MC ucode is missing on BTC parts.
5672 * The default clocks and voltages before the MC ucode
5673 * is loaded are not suffient for advanced operations.
5674 */
5675 if (ASIC_IS_DCE5(rdev)) {
5676 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5677 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5678 return -EINVAL;
5679 }
5680 }
5681
bcc1c2a1
AD		 5682 return 0;
5683}
5684
5685void evergreen_fini(struct radeon_device *rdev)
5686{
6c7bccea 5687 radeon_pm_fini(rdev);
7991d665 5688 radeon_audio_fini(rdev);
45f9a39b 5689 r700_cp_fini(rdev);
233d1ad5 5690 r600_dma_fini(rdev);
bcc1c2a1 5691 r600_irq_fini(rdev);
2948f5e6
AD		 5692 if (rdev->flags & RADEON_IS_IGP)
5693 sumo_rlc_fini(rdev);
724c80e1 5694 radeon_wb_fini(rdev);
2898c348 5695 radeon_ib_pool_fini(rdev);
bcc1c2a1 5696 radeon_irq_kms_fini(rdev);
e409b128 5697 uvd_v1_0_fini(rdev);
f2ba57b5 5698 radeon_uvd_fini(rdev);
d9654413 5699 evergreen_pcie_gart_fini(rdev);
16cdf04d 5700 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5701 radeon_gem_fini(rdev);
5702 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5703 radeon_agp_fini(rdev);
5704 radeon_bo_fini(rdev);
5705 radeon_atombios_fini(rdev);
5706 kfree(rdev->bios);
5707 rdev->bios = NULL;
bcc1c2a1 5708}
9e46a48d 5709
b07759bf 5710void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5711{
7e0e4196 5712 u32 link_width_cntl, speed_cntl;
9e46a48d 5713
d42dd579
AD		 5714 if (radeon_pcie_gen2 == 0)
5715 return;
5716
9e46a48d
AD		 5717 if (rdev->flags & RADEON_IS_IGP)
5718 return;
5719
5720 if (!(rdev->flags & RADEON_IS_PCIE))
5721 return;
5722
5723 /* x2 cards have a special sequence */
5724 if (ASIC_IS_X2(rdev))
5725 return;
5726
7e0e4196
KSS		 5727 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5728 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5729 return;
5730
492d2b61 5731 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5732 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5733 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5734 return;
5735 }
5736
197bbb3d
DA		 5737 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5738
9e46a48d
AD		 5739 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5740 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5741
492d2b61 5742 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5743 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5744 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5745
492d2b61 5746 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5747 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5748 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5749
492d2b61 5750 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5751 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5752 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5753
492d2b61 5754 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5755 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5756 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5757
492d2b61 5758 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5759 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5760 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5761
5762 } else {
492d2b61 5763 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5764 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5765 if (1)
5766 link_width_cntl |= LC_UPCONFIGURE_DIS;
5767 else
5768 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5769 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5770 }
5771}
f52382d7
AD		 5772
5773void evergreen_program_aspm(struct radeon_device *rdev)
5774{
5775 u32 data, orig;
5776 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5777 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5778 /* fusion_platform = true
5779 * if the system is a fusion system
5780 * (APU or DGPU in a fusion system).
5781 * todo: check if the system is a fusion platform.
5782 */
5783 bool fusion_platform = false;
5784
1294d4a3
AD		 5785 if (radeon_aspm == 0)
5786 return;
5787
f52382d7
AD		 5788 if (!(rdev->flags & RADEON_IS_PCIE))
5789 return;
5790
5791 switch (rdev->family) {
5792 case CHIP_CYPRESS:
5793 case CHIP_HEMLOCK:
5794 case CHIP_JUNIPER:
5795 case CHIP_REDWOOD:
5796 case CHIP_CEDAR:
5797 case CHIP_SUMO:
5798 case CHIP_SUMO2:
5799 case CHIP_PALM:
5800 case CHIP_ARUBA:
5801 disable_l0s = true;
5802 break;
5803 default:
5804 disable_l0s = false;
5805 break;
5806 }
5807
5808 if (rdev->flags & RADEON_IS_IGP)
5809 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5810
5811 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5812 if (fusion_platform)
5813 data &= ~MULTI_PIF;
5814 else
5815 data |= MULTI_PIF;
5816 if (data != orig)
5817 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5818
5819 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5820 if (fusion_platform)
5821 data &= ~MULTI_PIF;
5822 else
5823 data |= MULTI_PIF;
5824 if (data != orig)
5825 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5826
5827 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5828 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5829 if (!disable_l0s) {
5830 if (rdev->family >= CHIP_BARTS)
5831 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5832 else
5833 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5834 }
5835
5836 if (!disable_l1) {
5837 if (rdev->family >= CHIP_BARTS)
5838 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5839 else
5840 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5841
5842 if (!disable_plloff_in_l1) {
5843 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5844 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5845 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5846 if (data != orig)
5847 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5848
5849 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5850 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5851 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5852 if (data != orig)
5853 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5854
5855 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5856 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5857 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5858 if (data != orig)
5859 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5860
5861 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5862 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5863 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5864 if (data != orig)
5865 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5866
5867 if (rdev->family >= CHIP_BARTS) {
5868 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5869 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5870 data |= PLL_RAMP_UP_TIME_0(4);
5871 if (data != orig)
5872 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5873
5874 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5875 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5876 data |= PLL_RAMP_UP_TIME_1(4);
5877 if (data != orig)
5878 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5879
5880 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5881 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5882 data |= PLL_RAMP_UP_TIME_0(4);
5883 if (data != orig)
5884 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5885
5886 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5887 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5888 data |= PLL_RAMP_UP_TIME_1(4);
5889 if (data != orig)
5890 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5891 }
5892
5893 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5894 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5895 data |= LC_DYN_LANES_PWR_STATE(3);
5896 if (data != orig)
5897 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5898
5899 if (rdev->family >= CHIP_BARTS) {
5900 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5901 data &= ~LS2_EXIT_TIME_MASK;
5902 data |= LS2_EXIT_TIME(1);
5903 if (data != orig)
5904 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5905
5906 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5907 data &= ~LS2_EXIT_TIME_MASK;
5908 data |= LS2_EXIT_TIME(1);
5909 if (data != orig)
5910 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5911 }
5912 }
5913 }
5914
5915 /* evergreen parts only */
5916 if (rdev->family < CHIP_BARTS)
5917 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5918
5919 if (pcie_lc_cntl != pcie_lc_cntl_old)
5920 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5921}
Linux 6.x block layer and io_uring tree(s)