projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: Fix max_vblank_count value for current display engines
[linux-2.6-block.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
5a0e3ad6 25#include <linux/slab.h>
760285e7 26#include <drm/drmP.h>
bcc1c2a1 27#include "radeon.h"
e6990375 28#include "radeon_asic.h"
bfc1f97d 29#include "radeon_audio.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
4a15903d
AD		 38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
2948f5e6
AD		 48#include "clearstate_evergreen.h"
49
1fd11777 50static const u32 sumo_rlc_save_restore_register_list[] =
2948f5e6
AD		 51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
2948f5e6 134
bcc1c2a1
AD		 135static void evergreen_gpu_init(struct radeon_device *rdev);
136void evergreen_fini(struct radeon_device *rdev);
b07759bf 137void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 138void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 139extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
54e2e49c
AD		 141extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
22c775ce 143void cik_init_cp_pg_table(struct radeon_device *rdev);
bcc1c2a1 144
59a82d0e
AD		 145extern u32 si_get_csb_size(struct radeon_device *rdev);
146extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
a0f38609
AD		 147extern u32 cik_get_csb_size(struct radeon_device *rdev);
148extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
b5470b03 149extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
bcc1c2a1 150
d4788db3
AD		 151static const u32 evergreen_golden_registers[] =
152{
153 0x3f90, 0xffff0000, 0xff000000,
154 0x9148, 0xffff0000, 0xff000000,
155 0x3f94, 0xffff0000, 0xff000000,
156 0x914c, 0xffff0000, 0xff000000,
157 0x9b7c, 0xffffffff, 0x00000000,
158 0x8a14, 0xffffffff, 0x00000007,
159 0x8b10, 0xffffffff, 0x00000000,
160 0x960c, 0xffffffff, 0x54763210,
161 0x88c4, 0xffffffff, 0x000000c2,
162 0x88d4, 0xffffffff, 0x00000010,
163 0x8974, 0xffffffff, 0x00000000,
164 0xc78, 0x00000080, 0x00000080,
165 0x5eb4, 0xffffffff, 0x00000002,
166 0x5e78, 0xffffffff, 0x001000f0,
167 0x6104, 0x01000300, 0x00000000,
168 0x5bc0, 0x00300000, 0x00000000,
169 0x7030, 0xffffffff, 0x00000011,
170 0x7c30, 0xffffffff, 0x00000011,
171 0x10830, 0xffffffff, 0x00000011,
172 0x11430, 0xffffffff, 0x00000011,
173 0x12030, 0xffffffff, 0x00000011,
174 0x12c30, 0xffffffff, 0x00000011,
175 0xd02c, 0xffffffff, 0x08421000,
176 0x240c, 0xffffffff, 0x00000380,
177 0x8b24, 0xffffffff, 0x00ff0fff,
178 0x28a4c, 0x06000000, 0x06000000,
179 0x10c, 0x00000001, 0x00000001,
180 0x8d00, 0xffffffff, 0x100e4848,
181 0x8d04, 0xffffffff, 0x00164745,
182 0x8c00, 0xffffffff, 0xe4000003,
183 0x8c04, 0xffffffff, 0x40600060,
184 0x8c08, 0xffffffff, 0x001c001c,
185 0x8cf0, 0xffffffff, 0x08e00620,
186 0x8c20, 0xffffffff, 0x00800080,
187 0x8c24, 0xffffffff, 0x00800080,
188 0x8c18, 0xffffffff, 0x20202078,
189 0x8c1c, 0xffffffff, 0x00001010,
190 0x28350, 0xffffffff, 0x00000000,
191 0xa008, 0xffffffff, 0x00010000,
6abafb78 192 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 193 0x9508, 0xffffffff, 0x00000002,
194 0x913c, 0x0000000f, 0x0000000a
195};
196
197static const u32 evergreen_golden_registers2[] =
198{
199 0x2f4c, 0xffffffff, 0x00000000,
200 0x54f4, 0xffffffff, 0x00000000,
201 0x54f0, 0xffffffff, 0x00000000,
202 0x5498, 0xffffffff, 0x00000000,
203 0x549c, 0xffffffff, 0x00000000,
204 0x5494, 0xffffffff, 0x00000000,
205 0x53cc, 0xffffffff, 0x00000000,
206 0x53c8, 0xffffffff, 0x00000000,
207 0x53c4, 0xffffffff, 0x00000000,
208 0x53c0, 0xffffffff, 0x00000000,
209 0x53bc, 0xffffffff, 0x00000000,
210 0x53b8, 0xffffffff, 0x00000000,
211 0x53b4, 0xffffffff, 0x00000000,
212 0x53b0, 0xffffffff, 0x00000000
213};
214
215static const u32 cypress_mgcg_init[] =
216{
217 0x802c, 0xffffffff, 0xc0000000,
218 0x5448, 0xffffffff, 0x00000100,
219 0x55e4, 0xffffffff, 0x00000100,
220 0x160c, 0xffffffff, 0x00000100,
221 0x5644, 0xffffffff, 0x00000100,
222 0xc164, 0xffffffff, 0x00000100,
223 0x8a18, 0xffffffff, 0x00000100,
224 0x897c, 0xffffffff, 0x06000100,
225 0x8b28, 0xffffffff, 0x00000100,
226 0x9144, 0xffffffff, 0x00000100,
227 0x9a60, 0xffffffff, 0x00000100,
228 0x9868, 0xffffffff, 0x00000100,
229 0x8d58, 0xffffffff, 0x00000100,
230 0x9510, 0xffffffff, 0x00000100,
231 0x949c, 0xffffffff, 0x00000100,
232 0x9654, 0xffffffff, 0x00000100,
233 0x9030, 0xffffffff, 0x00000100,
234 0x9034, 0xffffffff, 0x00000100,
235 0x9038, 0xffffffff, 0x00000100,
236 0x903c, 0xffffffff, 0x00000100,
237 0x9040, 0xffffffff, 0x00000100,
238 0xa200, 0xffffffff, 0x00000100,
239 0xa204, 0xffffffff, 0x00000100,
240 0xa208, 0xffffffff, 0x00000100,
241 0xa20c, 0xffffffff, 0x00000100,
242 0x971c, 0xffffffff, 0x00000100,
243 0x977c, 0xffffffff, 0x00000100,
244 0x3f80, 0xffffffff, 0x00000100,
245 0xa210, 0xffffffff, 0x00000100,
246 0xa214, 0xffffffff, 0x00000100,
247 0x4d8, 0xffffffff, 0x00000100,
248 0x9784, 0xffffffff, 0x00000100,
249 0x9698, 0xffffffff, 0x00000100,
250 0x4d4, 0xffffffff, 0x00000200,
251 0x30cc, 0xffffffff, 0x00000100,
252 0xd0c0, 0xffffffff, 0xff000100,
253 0x802c, 0xffffffff, 0x40000000,
254 0x915c, 0xffffffff, 0x00010000,
255 0x9160, 0xffffffff, 0x00030002,
256 0x9178, 0xffffffff, 0x00070000,
257 0x917c, 0xffffffff, 0x00030002,
258 0x9180, 0xffffffff, 0x00050004,
259 0x918c, 0xffffffff, 0x00010006,
260 0x9190, 0xffffffff, 0x00090008,
261 0x9194, 0xffffffff, 0x00070000,
262 0x9198, 0xffffffff, 0x00030002,
263 0x919c, 0xffffffff, 0x00050004,
264 0x91a8, 0xffffffff, 0x00010006,
265 0x91ac, 0xffffffff, 0x00090008,
266 0x91b0, 0xffffffff, 0x00070000,
267 0x91b4, 0xffffffff, 0x00030002,
268 0x91b8, 0xffffffff, 0x00050004,
269 0x91c4, 0xffffffff, 0x00010006,
270 0x91c8, 0xffffffff, 0x00090008,
271 0x91cc, 0xffffffff, 0x00070000,
272 0x91d0, 0xffffffff, 0x00030002,
273 0x91d4, 0xffffffff, 0x00050004,
274 0x91e0, 0xffffffff, 0x00010006,
275 0x91e4, 0xffffffff, 0x00090008,
276 0x91e8, 0xffffffff, 0x00000000,
277 0x91ec, 0xffffffff, 0x00070000,
278 0x91f0, 0xffffffff, 0x00030002,
279 0x91f4, 0xffffffff, 0x00050004,
280 0x9200, 0xffffffff, 0x00010006,
281 0x9204, 0xffffffff, 0x00090008,
282 0x9208, 0xffffffff, 0x00070000,
283 0x920c, 0xffffffff, 0x00030002,
284 0x9210, 0xffffffff, 0x00050004,
285 0x921c, 0xffffffff, 0x00010006,
286 0x9220, 0xffffffff, 0x00090008,
287 0x9224, 0xffffffff, 0x00070000,
288 0x9228, 0xffffffff, 0x00030002,
289 0x922c, 0xffffffff, 0x00050004,
290 0x9238, 0xffffffff, 0x00010006,
291 0x923c, 0xffffffff, 0x00090008,
292 0x9240, 0xffffffff, 0x00070000,
293 0x9244, 0xffffffff, 0x00030002,
294 0x9248, 0xffffffff, 0x00050004,
295 0x9254, 0xffffffff, 0x00010006,
296 0x9258, 0xffffffff, 0x00090008,
297 0x925c, 0xffffffff, 0x00070000,
298 0x9260, 0xffffffff, 0x00030002,
299 0x9264, 0xffffffff, 0x00050004,
300 0x9270, 0xffffffff, 0x00010006,
301 0x9274, 0xffffffff, 0x00090008,
302 0x9278, 0xffffffff, 0x00070000,
303 0x927c, 0xffffffff, 0x00030002,
304 0x9280, 0xffffffff, 0x00050004,
305 0x928c, 0xffffffff, 0x00010006,
306 0x9290, 0xffffffff, 0x00090008,
307 0x9294, 0xffffffff, 0x00000000,
308 0x929c, 0xffffffff, 0x00000001,
309 0x802c, 0xffffffff, 0x40010000,
310 0x915c, 0xffffffff, 0x00010000,
311 0x9160, 0xffffffff, 0x00030002,
312 0x9178, 0xffffffff, 0x00070000,
313 0x917c, 0xffffffff, 0x00030002,
314 0x9180, 0xffffffff, 0x00050004,
315 0x918c, 0xffffffff, 0x00010006,
316 0x9190, 0xffffffff, 0x00090008,
317 0x9194, 0xffffffff, 0x00070000,
318 0x9198, 0xffffffff, 0x00030002,
319 0x919c, 0xffffffff, 0x00050004,
320 0x91a8, 0xffffffff, 0x00010006,
321 0x91ac, 0xffffffff, 0x00090008,
322 0x91b0, 0xffffffff, 0x00070000,
323 0x91b4, 0xffffffff, 0x00030002,
324 0x91b8, 0xffffffff, 0x00050004,
325 0x91c4, 0xffffffff, 0x00010006,
326 0x91c8, 0xffffffff, 0x00090008,
327 0x91cc, 0xffffffff, 0x00070000,
328 0x91d0, 0xffffffff, 0x00030002,
329 0x91d4, 0xffffffff, 0x00050004,
330 0x91e0, 0xffffffff, 0x00010006,
331 0x91e4, 0xffffffff, 0x00090008,
332 0x91e8, 0xffffffff, 0x00000000,
333 0x91ec, 0xffffffff, 0x00070000,
334 0x91f0, 0xffffffff, 0x00030002,
335 0x91f4, 0xffffffff, 0x00050004,
336 0x9200, 0xffffffff, 0x00010006,
337 0x9204, 0xffffffff, 0x00090008,
338 0x9208, 0xffffffff, 0x00070000,
339 0x920c, 0xffffffff, 0x00030002,
340 0x9210, 0xffffffff, 0x00050004,
341 0x921c, 0xffffffff, 0x00010006,
342 0x9220, 0xffffffff, 0x00090008,
343 0x9224, 0xffffffff, 0x00070000,
344 0x9228, 0xffffffff, 0x00030002,
345 0x922c, 0xffffffff, 0x00050004,
346 0x9238, 0xffffffff, 0x00010006,
347 0x923c, 0xffffffff, 0x00090008,
348 0x9240, 0xffffffff, 0x00070000,
349 0x9244, 0xffffffff, 0x00030002,
350 0x9248, 0xffffffff, 0x00050004,
351 0x9254, 0xffffffff, 0x00010006,
352 0x9258, 0xffffffff, 0x00090008,
353 0x925c, 0xffffffff, 0x00070000,
354 0x9260, 0xffffffff, 0x00030002,
355 0x9264, 0xffffffff, 0x00050004,
356 0x9270, 0xffffffff, 0x00010006,
357 0x9274, 0xffffffff, 0x00090008,
358 0x9278, 0xffffffff, 0x00070000,
359 0x927c, 0xffffffff, 0x00030002,
360 0x9280, 0xffffffff, 0x00050004,
361 0x928c, 0xffffffff, 0x00010006,
362 0x9290, 0xffffffff, 0x00090008,
363 0x9294, 0xffffffff, 0x00000000,
364 0x929c, 0xffffffff, 0x00000001,
365 0x802c, 0xffffffff, 0xc0000000
366};
367
368static const u32 redwood_mgcg_init[] =
369{
370 0x802c, 0xffffffff, 0xc0000000,
371 0x5448, 0xffffffff, 0x00000100,
372 0x55e4, 0xffffffff, 0x00000100,
373 0x160c, 0xffffffff, 0x00000100,
374 0x5644, 0xffffffff, 0x00000100,
375 0xc164, 0xffffffff, 0x00000100,
376 0x8a18, 0xffffffff, 0x00000100,
377 0x897c, 0xffffffff, 0x06000100,
378 0x8b28, 0xffffffff, 0x00000100,
379 0x9144, 0xffffffff, 0x00000100,
380 0x9a60, 0xffffffff, 0x00000100,
381 0x9868, 0xffffffff, 0x00000100,
382 0x8d58, 0xffffffff, 0x00000100,
383 0x9510, 0xffffffff, 0x00000100,
384 0x949c, 0xffffffff, 0x00000100,
385 0x9654, 0xffffffff, 0x00000100,
386 0x9030, 0xffffffff, 0x00000100,
387 0x9034, 0xffffffff, 0x00000100,
388 0x9038, 0xffffffff, 0x00000100,
389 0x903c, 0xffffffff, 0x00000100,
390 0x9040, 0xffffffff, 0x00000100,
391 0xa200, 0xffffffff, 0x00000100,
392 0xa204, 0xffffffff, 0x00000100,
393 0xa208, 0xffffffff, 0x00000100,
394 0xa20c, 0xffffffff, 0x00000100,
395 0x971c, 0xffffffff, 0x00000100,
396 0x977c, 0xffffffff, 0x00000100,
397 0x3f80, 0xffffffff, 0x00000100,
398 0xa210, 0xffffffff, 0x00000100,
399 0xa214, 0xffffffff, 0x00000100,
400 0x4d8, 0xffffffff, 0x00000100,
401 0x9784, 0xffffffff, 0x00000100,
402 0x9698, 0xffffffff, 0x00000100,
403 0x4d4, 0xffffffff, 0x00000200,
404 0x30cc, 0xffffffff, 0x00000100,
405 0xd0c0, 0xffffffff, 0xff000100,
406 0x802c, 0xffffffff, 0x40000000,
407 0x915c, 0xffffffff, 0x00010000,
408 0x9160, 0xffffffff, 0x00030002,
409 0x9178, 0xffffffff, 0x00070000,
410 0x917c, 0xffffffff, 0x00030002,
411 0x9180, 0xffffffff, 0x00050004,
412 0x918c, 0xffffffff, 0x00010006,
413 0x9190, 0xffffffff, 0x00090008,
414 0x9194, 0xffffffff, 0x00070000,
415 0x9198, 0xffffffff, 0x00030002,
416 0x919c, 0xffffffff, 0x00050004,
417 0x91a8, 0xffffffff, 0x00010006,
418 0x91ac, 0xffffffff, 0x00090008,
419 0x91b0, 0xffffffff, 0x00070000,
420 0x91b4, 0xffffffff, 0x00030002,
421 0x91b8, 0xffffffff, 0x00050004,
422 0x91c4, 0xffffffff, 0x00010006,
423 0x91c8, 0xffffffff, 0x00090008,
424 0x91cc, 0xffffffff, 0x00070000,
425 0x91d0, 0xffffffff, 0x00030002,
426 0x91d4, 0xffffffff, 0x00050004,
427 0x91e0, 0xffffffff, 0x00010006,
428 0x91e4, 0xffffffff, 0x00090008,
429 0x91e8, 0xffffffff, 0x00000000,
430 0x91ec, 0xffffffff, 0x00070000,
431 0x91f0, 0xffffffff, 0x00030002,
432 0x91f4, 0xffffffff, 0x00050004,
433 0x9200, 0xffffffff, 0x00010006,
434 0x9204, 0xffffffff, 0x00090008,
435 0x9294, 0xffffffff, 0x00000000,
436 0x929c, 0xffffffff, 0x00000001,
437 0x802c, 0xffffffff, 0xc0000000
438};
439
440static const u32 cedar_golden_registers[] =
441{
442 0x3f90, 0xffff0000, 0xff000000,
443 0x9148, 0xffff0000, 0xff000000,
444 0x3f94, 0xffff0000, 0xff000000,
445 0x914c, 0xffff0000, 0xff000000,
446 0x9b7c, 0xffffffff, 0x00000000,
447 0x8a14, 0xffffffff, 0x00000007,
448 0x8b10, 0xffffffff, 0x00000000,
449 0x960c, 0xffffffff, 0x54763210,
450 0x88c4, 0xffffffff, 0x000000c2,
451 0x88d4, 0xffffffff, 0x00000000,
452 0x8974, 0xffffffff, 0x00000000,
453 0xc78, 0x00000080, 0x00000080,
454 0x5eb4, 0xffffffff, 0x00000002,
455 0x5e78, 0xffffffff, 0x001000f0,
456 0x6104, 0x01000300, 0x00000000,
457 0x5bc0, 0x00300000, 0x00000000,
458 0x7030, 0xffffffff, 0x00000011,
459 0x7c30, 0xffffffff, 0x00000011,
460 0x10830, 0xffffffff, 0x00000011,
461 0x11430, 0xffffffff, 0x00000011,
462 0xd02c, 0xffffffff, 0x08421000,
463 0x240c, 0xffffffff, 0x00000380,
464 0x8b24, 0xffffffff, 0x00ff0fff,
465 0x28a4c, 0x06000000, 0x06000000,
466 0x10c, 0x00000001, 0x00000001,
467 0x8d00, 0xffffffff, 0x100e4848,
468 0x8d04, 0xffffffff, 0x00164745,
469 0x8c00, 0xffffffff, 0xe4000003,
470 0x8c04, 0xffffffff, 0x40600060,
471 0x8c08, 0xffffffff, 0x001c001c,
472 0x8cf0, 0xffffffff, 0x08e00410,
473 0x8c20, 0xffffffff, 0x00800080,
474 0x8c24, 0xffffffff, 0x00800080,
475 0x8c18, 0xffffffff, 0x20202078,
476 0x8c1c, 0xffffffff, 0x00001010,
477 0x28350, 0xffffffff, 0x00000000,
478 0xa008, 0xffffffff, 0x00010000,
6abafb78 479 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 480 0x9508, 0xffffffff, 0x00000002
481};
482
483static const u32 cedar_mgcg_init[] =
484{
485 0x802c, 0xffffffff, 0xc0000000,
486 0x5448, 0xffffffff, 0x00000100,
487 0x55e4, 0xffffffff, 0x00000100,
488 0x160c, 0xffffffff, 0x00000100,
489 0x5644, 0xffffffff, 0x00000100,
490 0xc164, 0xffffffff, 0x00000100,
491 0x8a18, 0xffffffff, 0x00000100,
492 0x897c, 0xffffffff, 0x06000100,
493 0x8b28, 0xffffffff, 0x00000100,
494 0x9144, 0xffffffff, 0x00000100,
495 0x9a60, 0xffffffff, 0x00000100,
496 0x9868, 0xffffffff, 0x00000100,
497 0x8d58, 0xffffffff, 0x00000100,
498 0x9510, 0xffffffff, 0x00000100,
499 0x949c, 0xffffffff, 0x00000100,
500 0x9654, 0xffffffff, 0x00000100,
501 0x9030, 0xffffffff, 0x00000100,
502 0x9034, 0xffffffff, 0x00000100,
503 0x9038, 0xffffffff, 0x00000100,
504 0x903c, 0xffffffff, 0x00000100,
505 0x9040, 0xffffffff, 0x00000100,
506 0xa200, 0xffffffff, 0x00000100,
507 0xa204, 0xffffffff, 0x00000100,
508 0xa208, 0xffffffff, 0x00000100,
509 0xa20c, 0xffffffff, 0x00000100,
510 0x971c, 0xffffffff, 0x00000100,
511 0x977c, 0xffffffff, 0x00000100,
512 0x3f80, 0xffffffff, 0x00000100,
513 0xa210, 0xffffffff, 0x00000100,
514 0xa214, 0xffffffff, 0x00000100,
515 0x4d8, 0xffffffff, 0x00000100,
516 0x9784, 0xffffffff, 0x00000100,
517 0x9698, 0xffffffff, 0x00000100,
518 0x4d4, 0xffffffff, 0x00000200,
519 0x30cc, 0xffffffff, 0x00000100,
520 0xd0c0, 0xffffffff, 0xff000100,
521 0x802c, 0xffffffff, 0x40000000,
522 0x915c, 0xffffffff, 0x00010000,
523 0x9178, 0xffffffff, 0x00050000,
524 0x917c, 0xffffffff, 0x00030002,
525 0x918c, 0xffffffff, 0x00010004,
526 0x9190, 0xffffffff, 0x00070006,
527 0x9194, 0xffffffff, 0x00050000,
528 0x9198, 0xffffffff, 0x00030002,
529 0x91a8, 0xffffffff, 0x00010004,
530 0x91ac, 0xffffffff, 0x00070006,
531 0x91e8, 0xffffffff, 0x00000000,
532 0x9294, 0xffffffff, 0x00000000,
533 0x929c, 0xffffffff, 0x00000001,
534 0x802c, 0xffffffff, 0xc0000000
535};
536
537static const u32 juniper_mgcg_init[] =
538{
539 0x802c, 0xffffffff, 0xc0000000,
540 0x5448, 0xffffffff, 0x00000100,
541 0x55e4, 0xffffffff, 0x00000100,
542 0x160c, 0xffffffff, 0x00000100,
543 0x5644, 0xffffffff, 0x00000100,
544 0xc164, 0xffffffff, 0x00000100,
545 0x8a18, 0xffffffff, 0x00000100,
546 0x897c, 0xffffffff, 0x06000100,
547 0x8b28, 0xffffffff, 0x00000100,
548 0x9144, 0xffffffff, 0x00000100,
549 0x9a60, 0xffffffff, 0x00000100,
550 0x9868, 0xffffffff, 0x00000100,
551 0x8d58, 0xffffffff, 0x00000100,
552 0x9510, 0xffffffff, 0x00000100,
553 0x949c, 0xffffffff, 0x00000100,
554 0x9654, 0xffffffff, 0x00000100,
555 0x9030, 0xffffffff, 0x00000100,
556 0x9034, 0xffffffff, 0x00000100,
557 0x9038, 0xffffffff, 0x00000100,
558 0x903c, 0xffffffff, 0x00000100,
559 0x9040, 0xffffffff, 0x00000100,
560 0xa200, 0xffffffff, 0x00000100,
561 0xa204, 0xffffffff, 0x00000100,
562 0xa208, 0xffffffff, 0x00000100,
563 0xa20c, 0xffffffff, 0x00000100,
564 0x971c, 0xffffffff, 0x00000100,
565 0xd0c0, 0xffffffff, 0xff000100,
566 0x802c, 0xffffffff, 0x40000000,
567 0x915c, 0xffffffff, 0x00010000,
568 0x9160, 0xffffffff, 0x00030002,
569 0x9178, 0xffffffff, 0x00070000,
570 0x917c, 0xffffffff, 0x00030002,
571 0x9180, 0xffffffff, 0x00050004,
572 0x918c, 0xffffffff, 0x00010006,
573 0x9190, 0xffffffff, 0x00090008,
574 0x9194, 0xffffffff, 0x00070000,
575 0x9198, 0xffffffff, 0x00030002,
576 0x919c, 0xffffffff, 0x00050004,
577 0x91a8, 0xffffffff, 0x00010006,
578 0x91ac, 0xffffffff, 0x00090008,
579 0x91b0, 0xffffffff, 0x00070000,
580 0x91b4, 0xffffffff, 0x00030002,
581 0x91b8, 0xffffffff, 0x00050004,
582 0x91c4, 0xffffffff, 0x00010006,
583 0x91c8, 0xffffffff, 0x00090008,
584 0x91cc, 0xffffffff, 0x00070000,
585 0x91d0, 0xffffffff, 0x00030002,
586 0x91d4, 0xffffffff, 0x00050004,
587 0x91e0, 0xffffffff, 0x00010006,
588 0x91e4, 0xffffffff, 0x00090008,
589 0x91e8, 0xffffffff, 0x00000000,
590 0x91ec, 0xffffffff, 0x00070000,
591 0x91f0, 0xffffffff, 0x00030002,
592 0x91f4, 0xffffffff, 0x00050004,
593 0x9200, 0xffffffff, 0x00010006,
594 0x9204, 0xffffffff, 0x00090008,
595 0x9208, 0xffffffff, 0x00070000,
596 0x920c, 0xffffffff, 0x00030002,
597 0x9210, 0xffffffff, 0x00050004,
598 0x921c, 0xffffffff, 0x00010006,
599 0x9220, 0xffffffff, 0x00090008,
600 0x9224, 0xffffffff, 0x00070000,
601 0x9228, 0xffffffff, 0x00030002,
602 0x922c, 0xffffffff, 0x00050004,
603 0x9238, 0xffffffff, 0x00010006,
604 0x923c, 0xffffffff, 0x00090008,
605 0x9240, 0xffffffff, 0x00070000,
606 0x9244, 0xffffffff, 0x00030002,
607 0x9248, 0xffffffff, 0x00050004,
608 0x9254, 0xffffffff, 0x00010006,
609 0x9258, 0xffffffff, 0x00090008,
610 0x925c, 0xffffffff, 0x00070000,
611 0x9260, 0xffffffff, 0x00030002,
612 0x9264, 0xffffffff, 0x00050004,
613 0x9270, 0xffffffff, 0x00010006,
614 0x9274, 0xffffffff, 0x00090008,
615 0x9278, 0xffffffff, 0x00070000,
616 0x927c, 0xffffffff, 0x00030002,
617 0x9280, 0xffffffff, 0x00050004,
618 0x928c, 0xffffffff, 0x00010006,
619 0x9290, 0xffffffff, 0x00090008,
620 0x9294, 0xffffffff, 0x00000000,
621 0x929c, 0xffffffff, 0x00000001,
622 0x802c, 0xffffffff, 0xc0000000,
623 0x977c, 0xffffffff, 0x00000100,
624 0x3f80, 0xffffffff, 0x00000100,
625 0xa210, 0xffffffff, 0x00000100,
626 0xa214, 0xffffffff, 0x00000100,
627 0x4d8, 0xffffffff, 0x00000100,
628 0x9784, 0xffffffff, 0x00000100,
629 0x9698, 0xffffffff, 0x00000100,
630 0x4d4, 0xffffffff, 0x00000200,
631 0x30cc, 0xffffffff, 0x00000100,
632 0x802c, 0xffffffff, 0xc0000000
633};
634
635static const u32 supersumo_golden_registers[] =
636{
637 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 638 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 639 0x7030, 0xffffffff, 0x00000011,
640 0x7c30, 0xffffffff, 0x00000011,
641 0x6104, 0x01000300, 0x00000000,
642 0x5bc0, 0x00300000, 0x00000000,
643 0x8c04, 0xffffffff, 0x40600060,
644 0x8c08, 0xffffffff, 0x001c001c,
645 0x8c20, 0xffffffff, 0x00800080,
646 0x8c24, 0xffffffff, 0x00800080,
647 0x8c18, 0xffffffff, 0x20202078,
648 0x8c1c, 0xffffffff, 0x00001010,
649 0x918c, 0xffffffff, 0x00010006,
650 0x91a8, 0xffffffff, 0x00010006,
651 0x91c4, 0xffffffff, 0x00010006,
652 0x91e0, 0xffffffff, 0x00010006,
653 0x9200, 0xffffffff, 0x00010006,
654 0x9150, 0xffffffff, 0x6e944040,
655 0x917c, 0xffffffff, 0x00030002,
656 0x9180, 0xffffffff, 0x00050004,
657 0x9198, 0xffffffff, 0x00030002,
658 0x919c, 0xffffffff, 0x00050004,
659 0x91b4, 0xffffffff, 0x00030002,
660 0x91b8, 0xffffffff, 0x00050004,
661 0x91d0, 0xffffffff, 0x00030002,
662 0x91d4, 0xffffffff, 0x00050004,
663 0x91f0, 0xffffffff, 0x00030002,
664 0x91f4, 0xffffffff, 0x00050004,
665 0x915c, 0xffffffff, 0x00010000,
666 0x9160, 0xffffffff, 0x00030002,
667 0x3f90, 0xffff0000, 0xff000000,
668 0x9178, 0xffffffff, 0x00070000,
669 0x9194, 0xffffffff, 0x00070000,
670 0x91b0, 0xffffffff, 0x00070000,
671 0x91cc, 0xffffffff, 0x00070000,
672 0x91ec, 0xffffffff, 0x00070000,
673 0x9148, 0xffff0000, 0xff000000,
674 0x9190, 0xffffffff, 0x00090008,
675 0x91ac, 0xffffffff, 0x00090008,
676 0x91c8, 0xffffffff, 0x00090008,
677 0x91e4, 0xffffffff, 0x00090008,
678 0x9204, 0xffffffff, 0x00090008,
679 0x3f94, 0xffff0000, 0xff000000,
680 0x914c, 0xffff0000, 0xff000000,
681 0x929c, 0xffffffff, 0x00000001,
682 0x8a18, 0xffffffff, 0x00000100,
683 0x8b28, 0xffffffff, 0x00000100,
684 0x9144, 0xffffffff, 0x00000100,
685 0x5644, 0xffffffff, 0x00000100,
686 0x9b7c, 0xffffffff, 0x00000000,
687 0x8030, 0xffffffff, 0x0000100a,
688 0x8a14, 0xffffffff, 0x00000007,
689 0x8b24, 0xffffffff, 0x00ff0fff,
690 0x8b10, 0xffffffff, 0x00000000,
691 0x28a4c, 0x06000000, 0x06000000,
692 0x4d8, 0xffffffff, 0x00000100,
693 0x913c, 0xffff000f, 0x0100000a,
694 0x960c, 0xffffffff, 0x54763210,
695 0x88c4, 0xffffffff, 0x000000c2,
696 0x88d4, 0xffffffff, 0x00000010,
697 0x8974, 0xffffffff, 0x00000000,
698 0xc78, 0x00000080, 0x00000080,
699 0x5e78, 0xffffffff, 0x001000f0,
700 0xd02c, 0xffffffff, 0x08421000,
701 0xa008, 0xffffffff, 0x00010000,
702 0x8d00, 0xffffffff, 0x100e4848,
703 0x8d04, 0xffffffff, 0x00164745,
704 0x8c00, 0xffffffff, 0xe4000003,
705 0x8cf0, 0x1fffffff, 0x08e00620,
706 0x28350, 0xffffffff, 0x00000000,
707 0x9508, 0xffffffff, 0x00000002
708};
709
710static const u32 sumo_golden_registers[] =
711{
712 0x900c, 0x00ffffff, 0x0017071f,
713 0x8c18, 0xffffffff, 0x10101060,
714 0x8c1c, 0xffffffff, 0x00001010,
715 0x8c30, 0x0000000f, 0x00000005,
716 0x9688, 0x0000000f, 0x00000007
717};
718
719static const u32 wrestler_golden_registers[] =
720{
721 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 722 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 723 0x7030, 0xffffffff, 0x00000011,
724 0x7c30, 0xffffffff, 0x00000011,
725 0x6104, 0x01000300, 0x00000000,
726 0x5bc0, 0x00300000, 0x00000000,
727 0x918c, 0xffffffff, 0x00010006,
728 0x91a8, 0xffffffff, 0x00010006,
729 0x9150, 0xffffffff, 0x6e944040,
730 0x917c, 0xffffffff, 0x00030002,
731 0x9198, 0xffffffff, 0x00030002,
732 0x915c, 0xffffffff, 0x00010000,
733 0x3f90, 0xffff0000, 0xff000000,
734 0x9178, 0xffffffff, 0x00070000,
735 0x9194, 0xffffffff, 0x00070000,
736 0x9148, 0xffff0000, 0xff000000,
737 0x9190, 0xffffffff, 0x00090008,
738 0x91ac, 0xffffffff, 0x00090008,
739 0x3f94, 0xffff0000, 0xff000000,
740 0x914c, 0xffff0000, 0xff000000,
741 0x929c, 0xffffffff, 0x00000001,
742 0x8a18, 0xffffffff, 0x00000100,
743 0x8b28, 0xffffffff, 0x00000100,
744 0x9144, 0xffffffff, 0x00000100,
745 0x9b7c, 0xffffffff, 0x00000000,
746 0x8030, 0xffffffff, 0x0000100a,
747 0x8a14, 0xffffffff, 0x00000001,
748 0x8b24, 0xffffffff, 0x00ff0fff,
749 0x8b10, 0xffffffff, 0x00000000,
750 0x28a4c, 0x06000000, 0x06000000,
751 0x4d8, 0xffffffff, 0x00000100,
752 0x913c, 0xffff000f, 0x0100000a,
753 0x960c, 0xffffffff, 0x54763210,
754 0x88c4, 0xffffffff, 0x000000c2,
755 0x88d4, 0xffffffff, 0x00000010,
756 0x8974, 0xffffffff, 0x00000000,
757 0xc78, 0x00000080, 0x00000080,
758 0x5e78, 0xffffffff, 0x001000f0,
759 0xd02c, 0xffffffff, 0x08421000,
760 0xa008, 0xffffffff, 0x00010000,
761 0x8d00, 0xffffffff, 0x100e4848,
762 0x8d04, 0xffffffff, 0x00164745,
763 0x8c00, 0xffffffff, 0xe4000003,
764 0x8cf0, 0x1fffffff, 0x08e00410,
765 0x28350, 0xffffffff, 0x00000000,
766 0x9508, 0xffffffff, 0x00000002,
767 0x900c, 0xffffffff, 0x0017071f,
768 0x8c18, 0xffffffff, 0x10101060,
769 0x8c1c, 0xffffffff, 0x00001010
770};
771
772static const u32 barts_golden_registers[] =
773{
774 0x5eb4, 0xffffffff, 0x00000002,
775 0x5e78, 0x8f311ff1, 0x001000f0,
776 0x3f90, 0xffff0000, 0xff000000,
777 0x9148, 0xffff0000, 0xff000000,
778 0x3f94, 0xffff0000, 0xff000000,
779 0x914c, 0xffff0000, 0xff000000,
780 0xc78, 0x00000080, 0x00000080,
781 0xbd4, 0x70073777, 0x00010001,
782 0xd02c, 0xbfffff1f, 0x08421000,
783 0xd0b8, 0x03773777, 0x02011003,
784 0x5bc0, 0x00200000, 0x50100000,
785 0x98f8, 0x33773777, 0x02011003,
786 0x98fc, 0xffffffff, 0x76543210,
787 0x7030, 0x31000311, 0x00000011,
788 0x2f48, 0x00000007, 0x02011003,
789 0x6b28, 0x00000010, 0x00000012,
790 0x7728, 0x00000010, 0x00000012,
791 0x10328, 0x00000010, 0x00000012,
792 0x10f28, 0x00000010, 0x00000012,
793 0x11b28, 0x00000010, 0x00000012,
794 0x12728, 0x00000010, 0x00000012,
795 0x240c, 0x000007ff, 0x00000380,
796 0x8a14, 0xf000001f, 0x00000007,
797 0x8b24, 0x3fff3fff, 0x00ff0fff,
798 0x8b10, 0x0000ff0f, 0x00000000,
799 0x28a4c, 0x07ffffff, 0x06000000,
800 0x10c, 0x00000001, 0x00010003,
801 0xa02c, 0xffffffff, 0x0000009b,
802 0x913c, 0x0000000f, 0x0100000a,
803 0x8d00, 0xffff7f7f, 0x100e4848,
804 0x8d04, 0x00ffffff, 0x00164745,
805 0x8c00, 0xfffc0003, 0xe4000003,
806 0x8c04, 0xf8ff00ff, 0x40600060,
807 0x8c08, 0x00ff00ff, 0x001c001c,
808 0x8cf0, 0x1fff1fff, 0x08e00620,
809 0x8c20, 0x0fff0fff, 0x00800080,
810 0x8c24, 0x0fff0fff, 0x00800080,
811 0x8c18, 0xffffffff, 0x20202078,
812 0x8c1c, 0x0000ffff, 0x00001010,
813 0x28350, 0x00000f01, 0x00000000,
814 0x9508, 0x3700001f, 0x00000002,
815 0x960c, 0xffffffff, 0x54763210,
816 0x88c4, 0x001f3ae3, 0x000000c2,
817 0x88d4, 0x0000001f, 0x00000010,
818 0x8974, 0xffffffff, 0x00000000
819};
820
821static const u32 turks_golden_registers[] =
822{
823 0x5eb4, 0xffffffff, 0x00000002,
824 0x5e78, 0x8f311ff1, 0x001000f0,
825 0x8c8, 0x00003000, 0x00001070,
826 0x8cc, 0x000fffff, 0x00040035,
827 0x3f90, 0xffff0000, 0xfff00000,
828 0x9148, 0xffff0000, 0xfff00000,
829 0x3f94, 0xffff0000, 0xfff00000,
830 0x914c, 0xffff0000, 0xfff00000,
831 0xc78, 0x00000080, 0x00000080,
832 0xbd4, 0x00073007, 0x00010002,
833 0xd02c, 0xbfffff1f, 0x08421000,
834 0xd0b8, 0x03773777, 0x02010002,
835 0x5bc0, 0x00200000, 0x50100000,
836 0x98f8, 0x33773777, 0x00010002,
837 0x98fc, 0xffffffff, 0x33221100,
838 0x7030, 0x31000311, 0x00000011,
839 0x2f48, 0x33773777, 0x00010002,
840 0x6b28, 0x00000010, 0x00000012,
841 0x7728, 0x00000010, 0x00000012,
842 0x10328, 0x00000010, 0x00000012,
843 0x10f28, 0x00000010, 0x00000012,
844 0x11b28, 0x00000010, 0x00000012,
845 0x12728, 0x00000010, 0x00000012,
846 0x240c, 0x000007ff, 0x00000380,
847 0x8a14, 0xf000001f, 0x00000007,
848 0x8b24, 0x3fff3fff, 0x00ff0fff,
849 0x8b10, 0x0000ff0f, 0x00000000,
850 0x28a4c, 0x07ffffff, 0x06000000,
851 0x10c, 0x00000001, 0x00010003,
852 0xa02c, 0xffffffff, 0x0000009b,
853 0x913c, 0x0000000f, 0x0100000a,
854 0x8d00, 0xffff7f7f, 0x100e4848,
855 0x8d04, 0x00ffffff, 0x00164745,
856 0x8c00, 0xfffc0003, 0xe4000003,
857 0x8c04, 0xf8ff00ff, 0x40600060,
858 0x8c08, 0x00ff00ff, 0x001c001c,
859 0x8cf0, 0x1fff1fff, 0x08e00410,
860 0x8c20, 0x0fff0fff, 0x00800080,
861 0x8c24, 0x0fff0fff, 0x00800080,
862 0x8c18, 0xffffffff, 0x20202078,
863 0x8c1c, 0x0000ffff, 0x00001010,
864 0x28350, 0x00000f01, 0x00000000,
865 0x9508, 0x3700001f, 0x00000002,
866 0x960c, 0xffffffff, 0x54763210,
867 0x88c4, 0x001f3ae3, 0x000000c2,
868 0x88d4, 0x0000001f, 0x00000010,
869 0x8974, 0xffffffff, 0x00000000
870};
871
872static const u32 caicos_golden_registers[] =
873{
874 0x5eb4, 0xffffffff, 0x00000002,
875 0x5e78, 0x8f311ff1, 0x001000f0,
876 0x8c8, 0x00003420, 0x00001450,
877 0x8cc, 0x000fffff, 0x00040035,
878 0x3f90, 0xffff0000, 0xfffc0000,
879 0x9148, 0xffff0000, 0xfffc0000,
880 0x3f94, 0xffff0000, 0xfffc0000,
881 0x914c, 0xffff0000, 0xfffc0000,
882 0xc78, 0x00000080, 0x00000080,
883 0xbd4, 0x00073007, 0x00010001,
884 0xd02c, 0xbfffff1f, 0x08421000,
885 0xd0b8, 0x03773777, 0x02010001,
886 0x5bc0, 0x00200000, 0x50100000,
887 0x98f8, 0x33773777, 0x02010001,
888 0x98fc, 0xffffffff, 0x33221100,
889 0x7030, 0x31000311, 0x00000011,
890 0x2f48, 0x33773777, 0x02010001,
891 0x6b28, 0x00000010, 0x00000012,
892 0x7728, 0x00000010, 0x00000012,
893 0x10328, 0x00000010, 0x00000012,
894 0x10f28, 0x00000010, 0x00000012,
895 0x11b28, 0x00000010, 0x00000012,
896 0x12728, 0x00000010, 0x00000012,
897 0x240c, 0x000007ff, 0x00000380,
898 0x8a14, 0xf000001f, 0x00000001,
899 0x8b24, 0x3fff3fff, 0x00ff0fff,
900 0x8b10, 0x0000ff0f, 0x00000000,
901 0x28a4c, 0x07ffffff, 0x06000000,
902 0x10c, 0x00000001, 0x00010003,
903 0xa02c, 0xffffffff, 0x0000009b,
904 0x913c, 0x0000000f, 0x0100000a,
905 0x8d00, 0xffff7f7f, 0x100e4848,
906 0x8d04, 0x00ffffff, 0x00164745,
907 0x8c00, 0xfffc0003, 0xe4000003,
908 0x8c04, 0xf8ff00ff, 0x40600060,
909 0x8c08, 0x00ff00ff, 0x001c001c,
910 0x8cf0, 0x1fff1fff, 0x08e00410,
911 0x8c20, 0x0fff0fff, 0x00800080,
912 0x8c24, 0x0fff0fff, 0x00800080,
913 0x8c18, 0xffffffff, 0x20202078,
914 0x8c1c, 0x0000ffff, 0x00001010,
915 0x28350, 0x00000f01, 0x00000000,
916 0x9508, 0x3700001f, 0x00000002,
917 0x960c, 0xffffffff, 0x54763210,
918 0x88c4, 0x001f3ae3, 0x000000c2,
919 0x88d4, 0x0000001f, 0x00000010,
920 0x8974, 0xffffffff, 0x00000000
921};
922
923static void evergreen_init_golden_registers(struct radeon_device *rdev)
924{
925 switch (rdev->family) {
926 case CHIP_CYPRESS:
927 case CHIP_HEMLOCK:
928 radeon_program_register_sequence(rdev,
929 evergreen_golden_registers,
930 (const u32)ARRAY_SIZE(evergreen_golden_registers));
931 radeon_program_register_sequence(rdev,
932 evergreen_golden_registers2,
933 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
934 radeon_program_register_sequence(rdev,
935 cypress_mgcg_init,
936 (const u32)ARRAY_SIZE(cypress_mgcg_init));
937 break;
938 case CHIP_JUNIPER:
939 radeon_program_register_sequence(rdev,
940 evergreen_golden_registers,
941 (const u32)ARRAY_SIZE(evergreen_golden_registers));
942 radeon_program_register_sequence(rdev,
943 evergreen_golden_registers2,
944 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
945 radeon_program_register_sequence(rdev,
946 juniper_mgcg_init,
947 (const u32)ARRAY_SIZE(juniper_mgcg_init));
948 break;
949 case CHIP_REDWOOD:
950 radeon_program_register_sequence(rdev,
951 evergreen_golden_registers,
952 (const u32)ARRAY_SIZE(evergreen_golden_registers));
953 radeon_program_register_sequence(rdev,
954 evergreen_golden_registers2,
955 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
956 radeon_program_register_sequence(rdev,
957 redwood_mgcg_init,
958 (const u32)ARRAY_SIZE(redwood_mgcg_init));
959 break;
960 case CHIP_CEDAR:
961 radeon_program_register_sequence(rdev,
962 cedar_golden_registers,
963 (const u32)ARRAY_SIZE(cedar_golden_registers));
964 radeon_program_register_sequence(rdev,
965 evergreen_golden_registers2,
966 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
967 radeon_program_register_sequence(rdev,
968 cedar_mgcg_init,
969 (const u32)ARRAY_SIZE(cedar_mgcg_init));
970 break;
971 case CHIP_PALM:
972 radeon_program_register_sequence(rdev,
973 wrestler_golden_registers,
974 (const u32)ARRAY_SIZE(wrestler_golden_registers));
975 break;
976 case CHIP_SUMO:
977 radeon_program_register_sequence(rdev,
978 supersumo_golden_registers,
979 (const u32)ARRAY_SIZE(supersumo_golden_registers));
980 break;
981 case CHIP_SUMO2:
982 radeon_program_register_sequence(rdev,
983 supersumo_golden_registers,
984 (const u32)ARRAY_SIZE(supersumo_golden_registers));
985 radeon_program_register_sequence(rdev,
986 sumo_golden_registers,
987 (const u32)ARRAY_SIZE(sumo_golden_registers));
988 break;
989 case CHIP_BARTS:
990 radeon_program_register_sequence(rdev,
991 barts_golden_registers,
992 (const u32)ARRAY_SIZE(barts_golden_registers));
993 break;
994 case CHIP_TURKS:
995 radeon_program_register_sequence(rdev,
996 turks_golden_registers,
997 (const u32)ARRAY_SIZE(turks_golden_registers));
998 break;
999 case CHIP_CAICOS:
1000 radeon_program_register_sequence(rdev,
1001 caicos_golden_registers,
1002 (const u32)ARRAY_SIZE(caicos_golden_registers));
1003 break;
1004 default:
1005 break;
1006 }
1007}
1008
ff609975
AD		 1009/**
1010 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1011 *
1012 * @rdev: radeon_device pointer
1013 * @reg: register offset in bytes
1014 * @val: register value
1015 *
1016 * Returns 0 for success or -EINVAL for an invalid register
1017 *
1018 */
1019int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1020 u32 reg, u32 *val)
1021{
1022 switch (reg) {
1023 case GRBM_STATUS:
1024 case GRBM_STATUS_SE0:
1025 case GRBM_STATUS_SE1:
1026 case SRBM_STATUS:
1027 case SRBM_STATUS2:
1028 case DMA_STATUS_REG:
1029 case UVD_STATUS:
1030 *val = RREG32(reg);
1031 return 0;
1032 default:
1033 return -EINVAL;
1034 }
1035}
1036
285484e2
JG		 1037void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1038 unsigned *bankh, unsigned *mtaspect,
1039 unsigned *tile_split)
1040{
1041 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1042 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1043 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1044 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1045 switch (*bankw) {
1046 default:
1047 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1048 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1049 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1050 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1051 }
1052 switch (*bankh) {
1053 default:
1054 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1055 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1056 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1057 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1058 }
1059 switch (*mtaspect) {
1060 default:
1061 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1062 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1063 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1064 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1065 }
1066}
1067
23d33ba3
AD		 1068static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1069 u32 cntl_reg, u32 status_reg)
1070{
1071 int r, i;
1072 struct atom_clock_dividers dividers;
1073
1074 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1075 clock, false, &dividers);
1076 if (r)
1077 return r;
1078
1079 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1080
1081 for (i = 0; i < 100; i++) {
1082 if (RREG32(status_reg) & DCLK_STATUS)
1083 break;
1084 mdelay(10);
1085 }
1086 if (i == 100)
1087 return -ETIMEDOUT;
1088
1089 return 0;
1090}
1091
1092int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1093{
1094 int r = 0;
1095 u32 cg_scratch = RREG32(CG_SCRATCH1);
1096
1097 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1098 if (r)
1099 goto done;
1100 cg_scratch &= 0xffff0000;
1101 cg_scratch |= vclk / 100; /* Mhz */
1102
1103 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1104 if (r)
1105 goto done;
1106 cg_scratch &= 0x0000ffff;
1107 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1108
1109done:
1110 WREG32(CG_SCRATCH1, cg_scratch);
1111
1112 return r;
1113}
1114
a8b4925c
AD		 1115int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1116{
1117 /* start off with something large */
facd112d 1118 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1119 int r;
1120
4ed10835
CK		 1121 /* bypass vclk and dclk with bclk */
1122 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1123 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1124 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1125
1126 /* put PLL in bypass mode */
1127 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1128
1129 if (!vclk || !dclk) {
1130 /* keep the Bypass mode, put PLL to sleep */
1131 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1132 return 0;
1133 }
1134
facd112d
CK		 1135 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1136 16384, 0x03FFFFFF, 0, 128, 5,
1137 &fb_div, &vclk_div, &dclk_div);
1138 if (r)
1139 return r;
a8b4925c
AD		 1140
1141 /* set VCO_MODE to 1 */
1142 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1143
1144 /* toggle UPLL_SLEEP to 1 then back to 0 */
1145 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1146 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1147
1148 /* deassert UPLL_RESET */
1149 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1150
1151 mdelay(1);
1152
facd112d 1153 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1154 if (r)
1155 return r;
1156
1157 /* assert UPLL_RESET again */
1158 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1159
1160 /* disable spread spectrum. */
1161 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1162
1163 /* set feedback divider */
facd112d 1164 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1165
1166 /* set ref divider to 0 */
1167 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1168
facd112d 1169 if (fb_div < 307200)
a8b4925c
AD		 1170 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1171 else
1172 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1173
1174 /* set PDIV_A and PDIV_B */
1175 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1176 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1177 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1178
1179 /* give the PLL some time to settle */
1180 mdelay(15);
1181
1182 /* deassert PLL_RESET */
1183 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1184
1185 mdelay(15);
1186
1187 /* switch from bypass mode to normal mode */
1188 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1189
facd112d 1190 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1191 if (r)
1192 return r;
1193
1194 /* switch VCLK and DCLK selection */
1195 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1196 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1197 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1198
1199 mdelay(100);
1200
1201 return 0;
1202}
1203
d054ac16
AD		 1204void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1205{
c11592fe
YW		 1206 int readrq;
1207 u16 v;
d054ac16 1208
c11592fe
YW		 1209 readrq = pcie_get_readrq(rdev->pdev);
1210 v = ffs(readrq) - 8;
d054ac16
AD		 1211 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1212 * to avoid hangs or perfomance issues
1213 */
c11592fe
YW		 1214 if ((v == 0) || (v == 6) || (v == 7))
1215 pcie_set_readrq(rdev->pdev, 512);
d054ac16
AD		 1216}
1217
134b480f
AD		 1218void dce4_program_fmt(struct drm_encoder *encoder)
1219{
1220 struct drm_device *dev = encoder->dev;
1221 struct radeon_device *rdev = dev->dev_private;
1222 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1223 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1224 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1225 int bpc = 0;
1226 u32 tmp = 0;
6214bb74 1227 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 1228
6214bb74
AD		 1229 if (connector) {
1230 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 1231 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 1232 dither = radeon_connector->dither;
1233 }
134b480f
AD		 1234
1235 /* LVDS/eDP FMT is set up by atom */
1236 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1237 return;
1238
1239 /* not needed for analog */
1240 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1241 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1242 return;
1243
1244 if (bpc == 0)
1245 return;
1246
1247 switch (bpc) {
1248 case 6:
6214bb74 1249 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1250 /* XXX sort out optimal dither settings */
1251 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1252 FMT_SPATIAL_DITHER_EN);
1253 else
1254 tmp |= FMT_TRUNCATE_EN;
1255 break;
1256 case 8:
6214bb74 1257 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1258 /* XXX sort out optimal dither settings */
1259 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1260 FMT_RGB_RANDOM_ENABLE |
1261 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1262 else
1263 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1264 break;
1265 case 10:
1266 default:
1267 /* not needed */
1268 break;
1269 }
1270
1271 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1272}
1273
10257a6d
AD		 1274static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1275{
1276 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1277 return true;
1278 else
1279 return false;
1280}
1281
1282static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1283{
1284 u32 pos1, pos2;
1285
1286 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1287 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1288
1289 if (pos1 != pos2)
1290 return true;
1291 else
1292 return false;
1293}
1294
377edc8b
AD		 1295/**
1296 * dce4_wait_for_vblank - vblank wait asic callback.
1297 *
1298 * @rdev: radeon_device pointer
1299 * @crtc: crtc to wait for vblank on
1300 *
1301 * Wait for vblank on the requested crtc (evergreen+).
1302 */
3ae19b75
AD		 1303void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1304{
10257a6d 1305 unsigned i = 0;
3ae19b75 1306
4a15903d
AD		 1307 if (crtc >= rdev->num_crtc)
1308 return;
1309
10257a6d
AD		 1310 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1311 return;
1312
1313 /* depending on when we hit vblank, we may be close to active; if so,
1314 * wait for another frame.
1315 */
1316 while (dce4_is_in_vblank(rdev, crtc)) {
1317 if (i++ % 100 == 0) {
1318 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1319 break;
3ae19b75 1320 }
10257a6d
AD		 1321 }
1322
1323 while (!dce4_is_in_vblank(rdev, crtc)) {
1324 if (i++ % 100 == 0) {
1325 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1326 break;
3ae19b75
AD		 1327 }
1328 }
1329}
1330
377edc8b
AD		 1331/**
1332 * evergreen_page_flip - pageflip callback.
1333 *
1334 * @rdev: radeon_device pointer
1335 * @crtc_id: crtc to cleanup pageflip on
1336 * @crtc_base: new address of the crtc (GPU MC address)
1337 *
1338 * Does the actual pageflip (evergreen+).
1339 * During vblank we take the crtc lock and wait for the update_pending
1340 * bit to go high, when it does, we release the lock, and allow the
1341 * double buffered update to take place.
1342 * Returns the current update pending status.
1343 */
157fa14d 1344void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
6f34be50
AD		 1345{
1346 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1347 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1348 int i;
6f34be50
AD		 1349
1350 /* Lock the graphics update lock */
1351 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1352 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1353
1354 /* update the scanout addresses */
1355 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1356 upper_32_bits(crtc_base));
1357 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1358 (u32)crtc_base);
1359
1360 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1361 upper_32_bits(crtc_base));
1362 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1363 (u32)crtc_base);
1364
1365 /* Wait for update_pending to go high. */
f6496479
AD		 1366 for (i = 0; i < rdev->usec_timeout; i++) {
1367 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1368 break;
1369 udelay(1);
1370 }
6f34be50
AD		 1371 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1372
1373 /* Unlock the lock, so double-buffering can take place inside vblank */
1374 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1375 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
157fa14d
CK		 1376}
1377
1378/**
1379 * evergreen_page_flip_pending - check if page flip is still pending
1380 *
1381 * @rdev: radeon_device pointer
1382 * @crtc_id: crtc to check
1383 *
1384 * Returns the current update pending status.
1385 */
1386bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1387{
1388 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
6f34be50
AD		 1389
1390 /* Return current update_pending status: */
157fa14d
CK		 1391 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1392 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
6f34be50
AD		 1393}
1394
21a8122a 1395/* get temperature in millidegrees */
20d391d7 1396int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1397{
1c88d74f
AD		 1398 u32 temp, toffset;
1399 int actual_temp = 0;
67b3f823
AD		 1400
1401 if (rdev->family == CHIP_JUNIPER) {
1402 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1403 TOFFSET_SHIFT;
1404 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1405 TS0_ADC_DOUT_SHIFT;
1406
1407 if (toffset & 0x100)
1408 actual_temp = temp / 2 - (0x200 - toffset);
1409 else
1410 actual_temp = temp / 2 + toffset;
1411
1412 actual_temp = actual_temp * 1000;
1413
1414 } else {
1415 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1416 ASIC_T_SHIFT;
1417
1418 if (temp & 0x400)
1419 actual_temp = -256;
1420 else if (temp & 0x200)
1421 actual_temp = 255;
1422 else if (temp & 0x100) {
1423 actual_temp = temp & 0x1ff;
1424 actual_temp |= ~0x1ff;
1425 } else
1426 actual_temp = temp & 0xff;
1427
1428 actual_temp = (actual_temp * 1000) / 2;
1429 }
21a8122a 1430
67b3f823 1431 return actual_temp;
21a8122a
AD		 1432}
1433
20d391d7 1434int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1435{
1436 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1437 int actual_temp = temp - 49;
e33df25f
AD		 1438
1439 return actual_temp * 1000;
1440}
1441
377edc8b
AD		 1442/**
1443 * sumo_pm_init_profile - Initialize power profiles callback.
1444 *
1445 * @rdev: radeon_device pointer
1446 *
1447 * Initialize the power states used in profile mode
1448 * (sumo, trinity, SI).
1449 * Used for profile mode only.
1450 */
a4c9e2ee
AD		 1451void sumo_pm_init_profile(struct radeon_device *rdev)
1452{
1453 int idx;
1454
1455 /* default */
1456 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1457 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1458 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1459 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1460
1461 /* low,mid sh/mh */
1462 if (rdev->flags & RADEON_IS_MOBILITY)
1463 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1464 else
1465 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1466
1467 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1469 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1470 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1471
1472 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1473 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1474 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1475 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1476
1477 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1478 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1479 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1480 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1481
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1483 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1484 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1485 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1486
1487 /* high sh/mh */
1488 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1489 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1490 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1491 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1492 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1493 rdev->pm.power_state[idx].num_clock_modes - 1;
1494
1495 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1496 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1497 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1498 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1499 rdev->pm.power_state[idx].num_clock_modes - 1;
1500}
1501
27810fb2
AD		 1502/**
1503 * btc_pm_init_profile - Initialize power profiles callback.
1504 *
1505 * @rdev: radeon_device pointer
1506 *
1507 * Initialize the power states used in profile mode
1508 * (BTC, cayman).
1509 * Used for profile mode only.
1510 */
1511void btc_pm_init_profile(struct radeon_device *rdev)
1512{
1513 int idx;
1514
1515 /* default */
1516 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1517 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1518 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1519 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1520 /* starting with BTC, there is one state that is used for both
1521 * MH and SH. Difference is that we always use the high clock index for
1522 * mclk.
1523 */
1524 if (rdev->flags & RADEON_IS_MOBILITY)
1525 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1526 else
1527 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1528 /* low sh */
1529 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1530 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1531 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1532 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1533 /* mid sh */
1534 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1535 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1536 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1537 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1538 /* high sh */
1539 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1540 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1541 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1542 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1543 /* low mh */
1544 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1545 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1546 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1547 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1548 /* mid mh */
1549 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1550 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1551 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1552 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1553 /* high mh */
1554 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1555 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1556 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1557 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1558}
1559
377edc8b
AD		 1560/**
1561 * evergreen_pm_misc - set additional pm hw parameters callback.
1562 *
1563 * @rdev: radeon_device pointer
1564 *
1565 * Set non-clock parameters associated with a power state
1566 * (voltage, etc.) (evergreen+).
1567 */
49e02b73
AD		 1568void evergreen_pm_misc(struct radeon_device *rdev)
1569{
a081a9d6
RM		 1570 int req_ps_idx = rdev->pm.requested_power_state_index;
1571 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1572 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1573 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1574
2feea49a 1575 if (voltage->type == VOLTAGE_SW) {
c6cf7777
AD		 1576 /* 0xff0x are flags rather then an actual voltage */
1577 if ((voltage->voltage & 0xff00) == 0xff00)
a377e187 1578 return;
2feea49a 1579 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1580 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1581 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1582 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1583 }
7ae764b1
AD		 1584
1585 /* starting with BTC, there is one state that is used for both
1586 * MH and SH. Difference is that we always use the high clock index for
1587 * mclk and vddci.
1588 */
1589 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1590 (rdev->family >= CHIP_BARTS) &&
1591 rdev->pm.active_crtc_count &&
1592 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1593 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1594 voltage = &rdev->pm.power_state[req_ps_idx].
1595 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1596
c6cf7777
AD		 1597 /* 0xff0x are flags rather then an actual voltage */
1598 if ((voltage->vddci & 0xff00) == 0xff00)
a377e187 1599 return;
2feea49a
AD		 1600 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1601 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1602 rdev->pm.current_vddci = voltage->vddci;
1603 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1604 }
1605 }
49e02b73
AD		 1606}
1607
377edc8b
AD		 1608/**
1609 * evergreen_pm_prepare - pre-power state change callback.
1610 *
1611 * @rdev: radeon_device pointer
1612 *
1613 * Prepare for a power state change (evergreen+).
1614 */
49e02b73
AD		 1615void evergreen_pm_prepare(struct radeon_device *rdev)
1616{
1617 struct drm_device *ddev = rdev->ddev;
1618 struct drm_crtc *crtc;
1619 struct radeon_crtc *radeon_crtc;
1620 u32 tmp;
1621
1622 /* disable any active CRTCs */
1623 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1624 radeon_crtc = to_radeon_crtc(crtc);
1625 if (radeon_crtc->enabled) {
1626 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1627 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1628 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1629 }
1630 }
1631}
1632
377edc8b
AD		 1633/**
1634 * evergreen_pm_finish - post-power state change callback.
1635 *
1636 * @rdev: radeon_device pointer
1637 *
1638 * Clean up after a power state change (evergreen+).
1639 */
49e02b73
AD		 1640void evergreen_pm_finish(struct radeon_device *rdev)
1641{
1642 struct drm_device *ddev = rdev->ddev;
1643 struct drm_crtc *crtc;
1644 struct radeon_crtc *radeon_crtc;
1645 u32 tmp;
1646
1647 /* enable any active CRTCs */
1648 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1649 radeon_crtc = to_radeon_crtc(crtc);
1650 if (radeon_crtc->enabled) {
1651 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1652 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1653 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1654 }
1655 }
1656}
1657
377edc8b
AD		 1658/**
1659 * evergreen_hpd_sense - hpd sense callback.
1660 *
1661 * @rdev: radeon_device pointer
1662 * @hpd: hpd (hotplug detect) pin
1663 *
1664 * Checks if a digital monitor is connected (evergreen+).
1665 * Returns true if connected, false if not connected.
1666 */
bcc1c2a1
AD		 1667bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1668{
1669 bool connected = false;
0ca2ab52
AD		 1670
1671 switch (hpd) {
1672 case RADEON_HPD_1:
1673 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1674 connected = true;
1675 break;
1676 case RADEON_HPD_2:
1677 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1678 connected = true;
1679 break;
1680 case RADEON_HPD_3:
1681 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1682 connected = true;
1683 break;
1684 case RADEON_HPD_4:
1685 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1686 connected = true;
1687 break;
1688 case RADEON_HPD_5:
1689 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1690 connected = true;
1691 break;
1692 case RADEON_HPD_6:
1693 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1694 connected = true;
ebc54ffe 1695 break;
0ca2ab52
AD		 1696 default:
1697 break;
1698 }
1699
bcc1c2a1
AD		 1700 return connected;
1701}
1702
377edc8b
AD		 1703/**
1704 * evergreen_hpd_set_polarity - hpd set polarity callback.
1705 *
1706 * @rdev: radeon_device pointer
1707 * @hpd: hpd (hotplug detect) pin
1708 *
1709 * Set the polarity of the hpd pin (evergreen+).
1710 */
bcc1c2a1
AD		 1711void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1712 enum radeon_hpd_id hpd)
1713{
0ca2ab52
AD		 1714 u32 tmp;
1715 bool connected = evergreen_hpd_sense(rdev, hpd);
1716
1717 switch (hpd) {
1718 case RADEON_HPD_1:
1719 tmp = RREG32(DC_HPD1_INT_CONTROL);
1720 if (connected)
1721 tmp &= ~DC_HPDx_INT_POLARITY;
1722 else
1723 tmp |= DC_HPDx_INT_POLARITY;
1724 WREG32(DC_HPD1_INT_CONTROL, tmp);
1725 break;
1726 case RADEON_HPD_2:
1727 tmp = RREG32(DC_HPD2_INT_CONTROL);
1728 if (connected)
1729 tmp &= ~DC_HPDx_INT_POLARITY;
1730 else
1731 tmp |= DC_HPDx_INT_POLARITY;
1732 WREG32(DC_HPD2_INT_CONTROL, tmp);
1733 break;
1734 case RADEON_HPD_3:
1735 tmp = RREG32(DC_HPD3_INT_CONTROL);
1736 if (connected)
1737 tmp &= ~DC_HPDx_INT_POLARITY;
1738 else
1739 tmp |= DC_HPDx_INT_POLARITY;
1740 WREG32(DC_HPD3_INT_CONTROL, tmp);
1741 break;
1742 case RADEON_HPD_4:
1743 tmp = RREG32(DC_HPD4_INT_CONTROL);
1744 if (connected)
1745 tmp &= ~DC_HPDx_INT_POLARITY;
1746 else
1747 tmp |= DC_HPDx_INT_POLARITY;
1748 WREG32(DC_HPD4_INT_CONTROL, tmp);
1749 break;
1750 case RADEON_HPD_5:
1751 tmp = RREG32(DC_HPD5_INT_CONTROL);
1752 if (connected)
1753 tmp &= ~DC_HPDx_INT_POLARITY;
1754 else
1755 tmp |= DC_HPDx_INT_POLARITY;
1756 WREG32(DC_HPD5_INT_CONTROL, tmp);
1757 break;
1758 case RADEON_HPD_6:
1759 tmp = RREG32(DC_HPD6_INT_CONTROL);
1760 if (connected)
1761 tmp &= ~DC_HPDx_INT_POLARITY;
1762 else
1763 tmp |= DC_HPDx_INT_POLARITY;
1764 WREG32(DC_HPD6_INT_CONTROL, tmp);
1765 break;
1766 default:
1767 break;
1768 }
bcc1c2a1
AD		 1769}
1770
377edc8b
AD		 1771/**
1772 * evergreen_hpd_init - hpd setup callback.
1773 *
1774 * @rdev: radeon_device pointer
1775 *
1776 * Setup the hpd pins used by the card (evergreen+).
1777 * Enable the pin, set the polarity, and enable the hpd interrupts.
1778 */
bcc1c2a1
AD		 1779void evergreen_hpd_init(struct radeon_device *rdev)
1780{
0ca2ab52
AD		 1781 struct drm_device *dev = rdev->ddev;
1782 struct drm_connector *connector;
fb98257a 1783 unsigned enabled = 0;
0ca2ab52
AD		 1784 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1785 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1786
0ca2ab52
AD		 1787 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1788 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1789
1790 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1791 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1792 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1793 * aux dp channel on imac and help (but not completely fix)
1794 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1795 * also avoid interrupt storms during dpms.
1796 */
1797 continue;
1798 }
0ca2ab52
AD		 1799 switch (radeon_connector->hpd.hpd) {
1800 case RADEON_HPD_1:
1801 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1802 break;
1803 case RADEON_HPD_2:
1804 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1805 break;
1806 case RADEON_HPD_3:
1807 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1808 break;
1809 case RADEON_HPD_4:
1810 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1811 break;
1812 case RADEON_HPD_5:
1813 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1814 break;
1815 case RADEON_HPD_6:
1816 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1817 break;
1818 default:
1819 break;
1820 }
64912e99 1821 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1822 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1823 }
fb98257a 1824 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1825}
1826
377edc8b
AD		 1827/**
1828 * evergreen_hpd_fini - hpd tear down callback.
1829 *
1830 * @rdev: radeon_device pointer
1831 *
1832 * Tear down the hpd pins used by the card (evergreen+).
1833 * Disable the hpd interrupts.
1834 */
0ca2ab52 1835void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1836{
0ca2ab52
AD		 1837 struct drm_device *dev = rdev->ddev;
1838 struct drm_connector *connector;
fb98257a 1839 unsigned disabled = 0;
0ca2ab52
AD		 1840
1841 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1842 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1843 switch (radeon_connector->hpd.hpd) {
1844 case RADEON_HPD_1:
1845 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1846 break;
1847 case RADEON_HPD_2:
1848 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1849 break;
1850 case RADEON_HPD_3:
1851 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1852 break;
1853 case RADEON_HPD_4:
1854 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1855 break;
1856 case RADEON_HPD_5:
1857 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1858 break;
1859 case RADEON_HPD_6:
1860 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1861 break;
1862 default:
1863 break;
1864 }
fb98257a 1865 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1866 }
fb98257a 1867 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1868}
1869
f9d9c362
AD		 1870/* watermark setup */
1871
1872static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1873 struct radeon_crtc *radeon_crtc,
1874 struct drm_display_mode *mode,
1875 struct drm_display_mode *other_mode)
1876{
0b31e023
AD		 1877 u32 tmp, buffer_alloc, i;
1878 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
f9d9c362
AD		 1879 /*
1880 * Line Buffer Setup
1881 * There are 3 line buffers, each one shared by 2 display controllers.
1882 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1883 * the display controllers. The paritioning is done via one of four
1884 * preset allocations specified in bits 2:0:
1885 * first display controller
1886 * 0 - first half of lb (3840 * 2)
1887 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1888 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1889 * 3 - first 1/4 of lb (1920 * 2)
1890 * second display controller
1891 * 4 - second half of lb (3840 * 2)
1892 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1893 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1894 * 7 - last 1/4 of lb (1920 * 2)
1895 */
12dfc843
AD		 1896 /* this can get tricky if we have two large displays on a paired group
1897 * of crtcs. Ideally for multiple large displays we'd assign them to
1898 * non-linked crtcs for maximum line buffer allocation.
1899 */
1900 if (radeon_crtc->base.enabled && mode) {
0b31e023 1901 if (other_mode) {
f9d9c362 1902 tmp = 0; /* 1/2 */
0b31e023
AD		 1903 buffer_alloc = 1;
1904 } else {
12dfc843 1905 tmp = 2; /* whole */
0b31e023
AD		 1906 buffer_alloc = 2;
1907 }
1908 } else {
12dfc843 1909 tmp = 0;
0b31e023
AD		 1910 buffer_alloc = 0;
1911 }
f9d9c362
AD		 1912
1913 /* second controller of the pair uses second half of the lb */
1914 if (radeon_crtc->crtc_id % 2)
1915 tmp += 4;
1916 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1917
0b31e023
AD		 1918 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1919 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1920 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1921 for (i = 0; i < rdev->usec_timeout; i++) {
1922 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1923 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1924 break;
1925 udelay(1);
1926 }
1927 }
1928
12dfc843
AD		 1929 if (radeon_crtc->base.enabled && mode) {
1930 switch (tmp) {
1931 case 0:
1932 case 4:
1933 default:
1934 if (ASIC_IS_DCE5(rdev))
1935 return 4096 * 2;
1936 else
1937 return 3840 * 2;
1938 case 1:
1939 case 5:
1940 if (ASIC_IS_DCE5(rdev))
1941 return 6144 * 2;
1942 else
1943 return 5760 * 2;
1944 case 2:
1945 case 6:
1946 if (ASIC_IS_DCE5(rdev))
1947 return 8192 * 2;
1948 else
1949 return 7680 * 2;
1950 case 3:
1951 case 7:
1952 if (ASIC_IS_DCE5(rdev))
1953 return 2048 * 2;
1954 else
1955 return 1920 * 2;
1956 }
f9d9c362 1957 }
12dfc843
AD		 1958
1959 /* controller not enabled, so no lb used */
1960 return 0;
f9d9c362
AD		 1961}
1962
ca7db22b 1963u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1964{
1965 u32 tmp = RREG32(MC_SHARED_CHMAP);
1966
1967 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1968 case 0:
1969 default:
1970 return 1;
1971 case 1:
1972 return 2;
1973 case 2:
1974 return 4;
1975 case 3:
1976 return 8;
1977 }
1978}
1979
1980struct evergreen_wm_params {
1981 u32 dram_channels; /* number of dram channels */
1982 u32 yclk; /* bandwidth per dram data pin in kHz */
1983 u32 sclk; /* engine clock in kHz */
1984 u32 disp_clk; /* display clock in kHz */
1985 u32 src_width; /* viewport width */
1986 u32 active_time; /* active display time in ns */
1987 u32 blank_time; /* blank time in ns */
1988 bool interlaced; /* mode is interlaced */
1989 fixed20_12 vsc; /* vertical scale ratio */
1990 u32 num_heads; /* number of active crtcs */
1991 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1992 u32 lb_size; /* line buffer allocated to pipe */
1993 u32 vtaps; /* vertical scaler taps */
1994};
1995
1996static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1997{
1998 /* Calculate DRAM Bandwidth and the part allocated to display. */
1999 fixed20_12 dram_efficiency; /* 0.7 */
2000 fixed20_12 yclk, dram_channels, bandwidth;
2001 fixed20_12 a;
2002
2003 a.full = dfixed_const(1000);
2004 yclk.full = dfixed_const(wm->yclk);
2005 yclk.full = dfixed_div(yclk, a);
2006 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2007 a.full = dfixed_const(10);
2008 dram_efficiency.full = dfixed_const(7);
2009 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2010 bandwidth.full = dfixed_mul(dram_channels, yclk);
2011 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2012
2013 return dfixed_trunc(bandwidth);
2014}
2015
2016static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2017{
2018 /* Calculate DRAM Bandwidth and the part allocated to display. */
2019 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2020 fixed20_12 yclk, dram_channels, bandwidth;
2021 fixed20_12 a;
2022
2023 a.full = dfixed_const(1000);
2024 yclk.full = dfixed_const(wm->yclk);
2025 yclk.full = dfixed_div(yclk, a);
2026 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2027 a.full = dfixed_const(10);
2028 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2029 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2030 bandwidth.full = dfixed_mul(dram_channels, yclk);
2031 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2032
2033 return dfixed_trunc(bandwidth);
2034}
2035
2036static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2037{
2038 /* Calculate the display Data return Bandwidth */
2039 fixed20_12 return_efficiency; /* 0.8 */
2040 fixed20_12 sclk, bandwidth;
2041 fixed20_12 a;
2042
2043 a.full = dfixed_const(1000);
2044 sclk.full = dfixed_const(wm->sclk);
2045 sclk.full = dfixed_div(sclk, a);
2046 a.full = dfixed_const(10);
2047 return_efficiency.full = dfixed_const(8);
2048 return_efficiency.full = dfixed_div(return_efficiency, a);
2049 a.full = dfixed_const(32);
2050 bandwidth.full = dfixed_mul(a, sclk);
2051 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2052
2053 return dfixed_trunc(bandwidth);
2054}
2055
2056static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2057{
2058 /* Calculate the DMIF Request Bandwidth */
2059 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2060 fixed20_12 disp_clk, bandwidth;
2061 fixed20_12 a;
2062
2063 a.full = dfixed_const(1000);
2064 disp_clk.full = dfixed_const(wm->disp_clk);
2065 disp_clk.full = dfixed_div(disp_clk, a);
2066 a.full = dfixed_const(10);
2067 disp_clk_request_efficiency.full = dfixed_const(8);
2068 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2069 a.full = dfixed_const(32);
2070 bandwidth.full = dfixed_mul(a, disp_clk);
2071 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2072
2073 return dfixed_trunc(bandwidth);
2074}
2075
2076static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2077{
2078 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2079 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2080 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2081 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2082
2083 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2084}
2085
2086static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2087{
2088 /* Calculate the display mode Average Bandwidth
2089 * DisplayMode should contain the source and destination dimensions,
2090 * timing, etc.
2091 */
2092 fixed20_12 bpp;
2093 fixed20_12 line_time;
2094 fixed20_12 src_width;
2095 fixed20_12 bandwidth;
2096 fixed20_12 a;
2097
2098 a.full = dfixed_const(1000);
2099 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2100 line_time.full = dfixed_div(line_time, a);
2101 bpp.full = dfixed_const(wm->bytes_per_pixel);
2102 src_width.full = dfixed_const(wm->src_width);
2103 bandwidth.full = dfixed_mul(src_width, bpp);
2104 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2105 bandwidth.full = dfixed_div(bandwidth, line_time);
2106
2107 return dfixed_trunc(bandwidth);
2108}
2109
2110static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2111{
2112 /* First calcualte the latency in ns */
2113 u32 mc_latency = 2000; /* 2000 ns. */
2114 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2115 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2116 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2117 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2118 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2119 (wm->num_heads * cursor_line_pair_return_time);
2120 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2121 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2122 fixed20_12 a, b, c;
2123
2124 if (wm->num_heads == 0)
2125 return 0;
2126
2127 a.full = dfixed_const(2);
2128 b.full = dfixed_const(1);
2129 if ((wm->vsc.full > a.full) ||
2130 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2131 (wm->vtaps >= 5) ||
2132 ((wm->vsc.full >= a.full) && wm->interlaced))
2133 max_src_lines_per_dst_line = 4;
2134 else
2135 max_src_lines_per_dst_line = 2;
2136
2137 a.full = dfixed_const(available_bandwidth);
2138 b.full = dfixed_const(wm->num_heads);
2139 a.full = dfixed_div(a, b);
2140
2141 b.full = dfixed_const(1000);
2142 c.full = dfixed_const(wm->disp_clk);
2143 b.full = dfixed_div(c, b);
2144 c.full = dfixed_const(wm->bytes_per_pixel);
2145 b.full = dfixed_mul(b, c);
2146
2147 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2148
2149 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2150 b.full = dfixed_const(1000);
2151 c.full = dfixed_const(lb_fill_bw);
2152 b.full = dfixed_div(c, b);
2153 a.full = dfixed_div(a, b);
2154 line_fill_time = dfixed_trunc(a);
2155
2156 if (line_fill_time < wm->active_time)
2157 return latency;
2158 else
2159 return latency + (line_fill_time - wm->active_time);
2160
2161}
2162
2163static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2164{
2165 if (evergreen_average_bandwidth(wm) <=
2166 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2167 return true;
2168 else
2169 return false;
2170};
2171
2172static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2173{
2174 if (evergreen_average_bandwidth(wm) <=
2175 (evergreen_available_bandwidth(wm) / wm->num_heads))
2176 return true;
2177 else
2178 return false;
2179};
2180
2181static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2182{
2183 u32 lb_partitions = wm->lb_size / wm->src_width;
2184 u32 line_time = wm->active_time + wm->blank_time;
2185 u32 latency_tolerant_lines;
2186 u32 latency_hiding;
2187 fixed20_12 a;
2188
2189 a.full = dfixed_const(1);
2190 if (wm->vsc.full > a.full)
2191 latency_tolerant_lines = 1;
2192 else {
2193 if (lb_partitions <= (wm->vtaps + 1))
2194 latency_tolerant_lines = 1;
2195 else
2196 latency_tolerant_lines = 2;
2197 }
2198
2199 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2200
2201 if (evergreen_latency_watermark(wm) <= latency_hiding)
2202 return true;
2203 else
2204 return false;
2205}
2206
2207static void evergreen_program_watermarks(struct radeon_device *rdev,
2208 struct radeon_crtc *radeon_crtc,
2209 u32 lb_size, u32 num_heads)
2210{
2211 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2212 struct evergreen_wm_params wm_low, wm_high;
2213 u32 dram_channels;
f9d9c362
AD		 2214 u32 pixel_period;
2215 u32 line_time = 0;
2216 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2217 u32 priority_a_mark = 0, priority_b_mark = 0;
2218 u32 priority_a_cnt = PRIORITY_OFF;
2219 u32 priority_b_cnt = PRIORITY_OFF;
2220 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2221 u32 tmp, arb_control3;
2222 fixed20_12 a, b, c;
2223
2224 if (radeon_crtc->base.enabled && num_heads && mode) {
2225 pixel_period = 1000000 / (u32)mode->clock;
2226 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2227 priority_a_cnt = 0;
2228 priority_b_cnt = 0;
cf0cfdd7
AD		 2229 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2230
2231 /* watermark for high clocks */
2232 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2233 wm_high.yclk =
2234 radeon_dpm_get_mclk(rdev, false) * 10;
2235 wm_high.sclk =
2236 radeon_dpm_get_sclk(rdev, false) * 10;
2237 } else {
2238 wm_high.yclk = rdev->pm.current_mclk * 10;
2239 wm_high.sclk = rdev->pm.current_sclk * 10;
2240 }
f9d9c362 2241
cf0cfdd7
AD		 2242 wm_high.disp_clk = mode->clock;
2243 wm_high.src_width = mode->crtc_hdisplay;
2244 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2245 wm_high.blank_time = line_time - wm_high.active_time;
2246 wm_high.interlaced = false;
f9d9c362 2247 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2248 wm_high.interlaced = true;
2249 wm_high.vsc = radeon_crtc->vsc;
2250 wm_high.vtaps = 1;
f9d9c362 2251 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2252 wm_high.vtaps = 2;
2253 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2254 wm_high.lb_size = lb_size;
2255 wm_high.dram_channels = dram_channels;
2256 wm_high.num_heads = num_heads;
2257
2258 /* watermark for low clocks */
2259 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2260 wm_low.yclk =
2261 radeon_dpm_get_mclk(rdev, true) * 10;
2262 wm_low.sclk =
2263 radeon_dpm_get_sclk(rdev, true) * 10;
2264 } else {
2265 wm_low.yclk = rdev->pm.current_mclk * 10;
2266 wm_low.sclk = rdev->pm.current_sclk * 10;
2267 }
2268
2269 wm_low.disp_clk = mode->clock;
2270 wm_low.src_width = mode->crtc_hdisplay;
2271 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2272 wm_low.blank_time = line_time - wm_low.active_time;
2273 wm_low.interlaced = false;
2274 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2275 wm_low.interlaced = true;
2276 wm_low.vsc = radeon_crtc->vsc;
2277 wm_low.vtaps = 1;
2278 if (radeon_crtc->rmx_type != RMX_OFF)
2279 wm_low.vtaps = 2;
2280 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2281 wm_low.lb_size = lb_size;
2282 wm_low.dram_channels = dram_channels;
2283 wm_low.num_heads = num_heads;
f9d9c362
AD		 2284
2285 /* set for high clocks */
cf0cfdd7 2286 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2287 /* set for low clocks */
cf0cfdd7 2288 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2289
2290 /* possibly force display priority to high */
2291 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2292 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2293 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2294 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2295 (rdev->disp_priority == 2)) {
cf0cfdd7 2296 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2297 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2298 }
2299 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2300 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2301 !evergreen_check_latency_hiding(&wm_low) ||
2302 (rdev->disp_priority == 2)) {
2303 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2304 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2305 }
2306
2307 a.full = dfixed_const(1000);
2308 b.full = dfixed_const(mode->clock);
2309 b.full = dfixed_div(b, a);
2310 c.full = dfixed_const(latency_watermark_a);
2311 c.full = dfixed_mul(c, b);
2312 c.full = dfixed_mul(c, radeon_crtc->hsc);
2313 c.full = dfixed_div(c, a);
2314 a.full = dfixed_const(16);
2315 c.full = dfixed_div(c, a);
2316 priority_a_mark = dfixed_trunc(c);
2317 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2318
2319 a.full = dfixed_const(1000);
2320 b.full = dfixed_const(mode->clock);
2321 b.full = dfixed_div(b, a);
2322 c.full = dfixed_const(latency_watermark_b);
2323 c.full = dfixed_mul(c, b);
2324 c.full = dfixed_mul(c, radeon_crtc->hsc);
2325 c.full = dfixed_div(c, a);
2326 a.full = dfixed_const(16);
2327 c.full = dfixed_div(c, a);
2328 priority_b_mark = dfixed_trunc(c);
2329 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2330 }
2331
2332 /* select wm A */
2333 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2334 tmp = arb_control3;
2335 tmp &= ~LATENCY_WATERMARK_MASK(3);
2336 tmp |= LATENCY_WATERMARK_MASK(1);
2337 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2338 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2339 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2340 LATENCY_HIGH_WATERMARK(line_time)));
2341 /* select wm B */
2342 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2343 tmp &= ~LATENCY_WATERMARK_MASK(3);
2344 tmp |= LATENCY_WATERMARK_MASK(2);
2345 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2346 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2347 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2348 LATENCY_HIGH_WATERMARK(line_time)));
2349 /* restore original selection */
2350 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2351
2352 /* write the priority marks */
2353 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2354 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2355
7178d2a6
AD		 2356 /* save values for DPM */
2357 radeon_crtc->line_time = line_time;
2358 radeon_crtc->wm_high = latency_watermark_a;
2359 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2360}
2361
377edc8b
AD		 2362/**
2363 * evergreen_bandwidth_update - update display watermarks callback.
2364 *
2365 * @rdev: radeon_device pointer
2366 *
2367 * Update the display watermarks based on the requested mode(s)
2368 * (evergreen+).
2369 */
0ca2ab52 2370void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2371{
f9d9c362
AD		 2372 struct drm_display_mode *mode0 = NULL;
2373 struct drm_display_mode *mode1 = NULL;
2374 u32 num_heads = 0, lb_size;
2375 int i;
2376
8efe82ca
AD		 2377 if (!rdev->mode_info.mode_config_initialized)
2378 return;
2379
f9d9c362
AD		 2380 radeon_update_display_priority(rdev);
2381
2382 for (i = 0; i < rdev->num_crtc; i++) {
2383 if (rdev->mode_info.crtcs[i]->base.enabled)
2384 num_heads++;
2385 }
2386 for (i = 0; i < rdev->num_crtc; i += 2) {
2387 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2388 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2389 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2390 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2391 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2392 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2393 }
bcc1c2a1
AD		 2394}
2395
377edc8b
AD		 2396/**
2397 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2398 *
2399 * @rdev: radeon_device pointer
2400 *
2401 * Wait for the MC (memory controller) to be idle.
2402 * (evergreen+).
2403 * Returns 0 if the MC is idle, -1 if not.
2404 */
b9952a8a 2405int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2406{
2407 unsigned i;
2408 u32 tmp;
2409
2410 for (i = 0; i < rdev->usec_timeout; i++) {
2411 /* read MC_STATUS */
2412 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2413 if (!tmp)
2414 return 0;
2415 udelay(1);
2416 }
2417 return -1;
2418}
2419
2420/*
2421 * GART
2422 */
0fcdb61e
AD		 2423void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2424{
2425 unsigned i;
2426 u32 tmp;
2427
6f2f48a9
AD		 2428 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2429
0fcdb61e
AD		 2430 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2431 for (i = 0; i < rdev->usec_timeout; i++) {
2432 /* read MC_STATUS */
2433 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2434 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2435 if (tmp == 2) {
2436 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2437 return;
2438 }
2439 if (tmp) {
2440 return;
2441 }
2442 udelay(1);
2443 }
2444}
2445
1109ca09 2446static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2447{
2448 u32 tmp;
0fcdb61e 2449 int r;
bcc1c2a1 2450
c9a1be96 2451 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2452 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2453 return -EINVAL;
2454 }
2455 r = radeon_gart_table_vram_pin(rdev);
2456 if (r)
2457 return r;
2458 /* Setup L2 cache */
2459 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2460 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2461 EFFECTIVE_L2_QUEUE_SIZE(7));
2462 WREG32(VM_L2_CNTL2, 0);
2463 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2464 /* Setup TLB control */
2465 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2466 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2467 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2468 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2469 if (rdev->flags & RADEON_IS_IGP) {
2470 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2471 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2472 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2473 } else {
2474 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2475 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2476 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2477 if ((rdev->family == CHIP_JUNIPER) ||
2478 (rdev->family == CHIP_CYPRESS) ||
2479 (rdev->family == CHIP_HEMLOCK) ||
2480 (rdev->family == CHIP_BARTS))
2481 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2482 }
bcc1c2a1
AD		 2483 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2484 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2485 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2486 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2487 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
607d4806 2488 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, (rdev->mc.gtt_end >> 12) - 1);
bcc1c2a1
AD		 2489 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2490 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2491 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2492 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2493 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2494 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2495
0fcdb61e 2496 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2497 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2498 (unsigned)(rdev->mc.gtt_size >> 20),
2499 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2500 rdev->gart.ready = true;
2501 return 0;
2502}
2503
1109ca09 2504static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2505{
2506 u32 tmp;
bcc1c2a1
AD		 2507
2508 /* Disable all tables */
0fcdb61e
AD		 2509 WREG32(VM_CONTEXT0_CNTL, 0);
2510 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2511
2512 /* Setup L2 cache */
2513 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2514 EFFECTIVE_L2_QUEUE_SIZE(7));
2515 WREG32(VM_L2_CNTL2, 0);
2516 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2517 /* Setup TLB control */
2518 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2519 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2520 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2521 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2522 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2523 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2524 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2525 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2526 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2527}
2528
1109ca09 2529static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2530{
2531 evergreen_pcie_gart_disable(rdev);
2532 radeon_gart_table_vram_free(rdev);
2533 radeon_gart_fini(rdev);
2534}
2535
2536
1109ca09 2537static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2538{
2539 u32 tmp;
bcc1c2a1
AD		 2540
2541 /* Setup L2 cache */
2542 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2543 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2544 EFFECTIVE_L2_QUEUE_SIZE(7));
2545 WREG32(VM_L2_CNTL2, 0);
2546 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2547 /* Setup TLB control */
2548 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2549 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2550 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2551 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2552 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2553 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2554 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2555 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2556 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2557 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2558 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2559 WREG32(VM_CONTEXT0_CNTL, 0);
2560 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2561}
2562
b9952a8a 2563void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2564{
62444b74
AD		 2565 u32 crtc_enabled, tmp, frame_count, blackout;
2566 int i, j;
2567
5153550a
AD		 2568 if (!ASIC_IS_NODCE(rdev)) {
2569 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2570 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2571
5153550a
AD		 2572 /* disable VGA render */
2573 WREG32(VGA_RENDER_CONTROL, 0);
2574 }
62444b74
AD		 2575 /* blank the display controllers */
2576 for (i = 0; i < rdev->num_crtc; i++) {
2577 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2578 if (crtc_enabled) {
2579 save->crtc_enabled[i] = true;
2580 if (ASIC_IS_DCE6(rdev)) {
2581 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2582 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2583 radeon_wait_for_vblank(rdev, i);
abf1457b 2584 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2585 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2586 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
f0d7bfb9 2587 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2588 }
2589 } else {
2590 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2591 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2592 radeon_wait_for_vblank(rdev, i);
abf1457b 2593 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2594 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2595 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2596 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2597 }
2598 }
2599 /* wait for the next frame */
2600 frame_count = radeon_get_vblank_counter(rdev, i);
2601 for (j = 0; j < rdev->usec_timeout; j++) {
2602 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2603 break;
2604 udelay(1);
2605 }
abf1457b
AD		 2606
2607 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2608 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2609 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2610 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2611 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2612 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2613 save->crtc_enabled[i] = false;
2614 /* ***** */
804cc4a0
AD		 2615 } else {
2616 save->crtc_enabled[i] = false;
62444b74 2617 }
18007401 2618 }
bcc1c2a1 2619
62444b74
AD		 2620 radeon_mc_wait_for_idle(rdev);
2621
2622 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2623 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2624 /* Block CPU access */
2625 WREG32(BIF_FB_EN, 0);
2626 /* blackout the MC */
2627 blackout &= ~BLACKOUT_MODE_MASK;
2628 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2629 }
ed39fadd
AD		 2630 /* wait for the MC to settle */
2631 udelay(100);
968c0166
AD		 2632
2633 /* lock double buffered regs */
2634 for (i = 0; i < rdev->num_crtc; i++) {
2635 if (save->crtc_enabled[i]) {
2636 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2637 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2638 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2639 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2640 }
2641 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2642 if (!(tmp & 1)) {
2643 tmp |= 1;
2644 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2645 }
2646 }
2647 }
bcc1c2a1
AD		 2648}
2649
b9952a8a 2650void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2651{
62444b74
AD		 2652 u32 tmp, frame_count;
2653 int i, j;
18007401 2654
62444b74
AD		 2655 /* update crtc base addresses */
2656 for (i = 0; i < rdev->num_crtc; i++) {
2657 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2658 upper_32_bits(rdev->mc.vram_start));
62444b74 2659 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2660 upper_32_bits(rdev->mc.vram_start));
62444b74 2661 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2662 (u32)rdev->mc.vram_start);
62444b74 2663 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2664 (u32)rdev->mc.vram_start);
2665 }
5153550a
AD		 2666
2667 if (!ASIC_IS_NODCE(rdev)) {
2668 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2669 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2670 }
62444b74 2671
968c0166
AD		 2672 /* unlock regs and wait for update */
2673 for (i = 0; i < rdev->num_crtc; i++) {
2674 if (save->crtc_enabled[i]) {
2675 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
f53f81b2
MK		 2676 if ((tmp & 0x7) != 3) {
2677 tmp &= ~0x7;
2678 tmp |= 0x3;
968c0166
AD		 2679 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2680 }
2681 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2682 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2683 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2684 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2685 }
2686 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2687 if (tmp & 1) {
2688 tmp &= ~1;
2689 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2690 }
2691 for (j = 0; j < rdev->usec_timeout; j++) {
2692 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2693 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2694 break;
2695 udelay(1);
2696 }
2697 }
2698 }
2699
62444b74
AD		 2700 /* unblackout the MC */
2701 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2702 tmp &= ~BLACKOUT_MODE_MASK;
2703 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2704 /* allow CPU access */
2705 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2706
2707 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2708 if (save->crtc_enabled[i]) {
62444b74
AD		 2709 if (ASIC_IS_DCE6(rdev)) {
2710 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
3157c589 2711 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2712 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2713 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2714 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2715 } else {
2716 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2717 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2718 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2719 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2720 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2721 }
2722 /* wait for the next frame */
2723 frame_count = radeon_get_vblank_counter(rdev, i);
2724 for (j = 0; j < rdev->usec_timeout; j++) {
2725 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2726 break;
2727 udelay(1);
2728 }
2729 }
2730 }
5153550a
AD		 2731 if (!ASIC_IS_NODCE(rdev)) {
2732 /* Unlock vga access */
2733 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2734 mdelay(1);
2735 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2736 }
bcc1c2a1
AD		 2737}
2738
755d819e 2739void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2740{
2741 struct evergreen_mc_save save;
2742 u32 tmp;
2743 int i, j;
2744
2745 /* Initialize HDP */
2746 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2747 WREG32((0x2c14 + j), 0x00000000);
2748 WREG32((0x2c18 + j), 0x00000000);
2749 WREG32((0x2c1c + j), 0x00000000);
2750 WREG32((0x2c20 + j), 0x00000000);
2751 WREG32((0x2c24 + j), 0x00000000);
2752 }
2753 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2754
2755 evergreen_mc_stop(rdev, &save);
2756 if (evergreen_mc_wait_for_idle(rdev)) {
2757 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2758 }
2759 /* Lockout access through VGA aperture*/
2760 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2761 /* Update configuration */
2762 if (rdev->flags & RADEON_IS_AGP) {
2763 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2764 /* VRAM before AGP */
2765 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2766 rdev->mc.vram_start >> 12);
2767 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2768 rdev->mc.gtt_end >> 12);
2769 } else {
2770 /* VRAM after AGP */
2771 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2772 rdev->mc.gtt_start >> 12);
2773 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2774 rdev->mc.vram_end >> 12);
2775 }
2776 } else {
2777 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2778 rdev->mc.vram_start >> 12);
2779 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2780 rdev->mc.vram_end >> 12);
2781 }
3b9832f6 2782 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2783 /* llano/ontario only */
2784 if ((rdev->family == CHIP_PALM) ||
2785 (rdev->family == CHIP_SUMO) ||
2786 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2787 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2788 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2789 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2790 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2791 }
bcc1c2a1
AD		 2792 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2793 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2794 WREG32(MC_VM_FB_LOCATION, tmp);
2795 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2796 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2797 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2798 if (rdev->flags & RADEON_IS_AGP) {
2799 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2800 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2801 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2802 } else {
2803 WREG32(MC_VM_AGP_BASE, 0);
2804 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2805 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2806 }
2807 if (evergreen_mc_wait_for_idle(rdev)) {
2808 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2809 }
2810 evergreen_mc_resume(rdev, &save);
2811 /* we need to own VRAM, so turn off the VGA renderer here
2812 * to stop it overwriting our objects */
2813 rv515_vga_render_disable(rdev);
2814}
2815
bcc1c2a1
AD		 2816/*
2817 * CP.
2818 */
12920591
AD		 2819void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2820{
876dc9f3 2821 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2822 u32 next_rptr;
7b1f2485 2823
12920591 2824 /* set to DX10/11 mode */
e32eb50d
CK		 2825 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2826 radeon_ring_write(ring, 1);
45df6803
CK		 2827
2828 if (ring->rptr_save_reg) {
89d35807 2829 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2830 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2831 radeon_ring_write(ring, ((ring->rptr_save_reg -
2832 PACKET3_SET_CONFIG_REG_START) >> 2));
2833 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2834 } else if (rdev->wb.enabled) {
2835 next_rptr = ring->wptr + 5 + 4;
2836 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2837 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2838 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2839 radeon_ring_write(ring, next_rptr);
2840 radeon_ring_write(ring, 0);
45df6803
CK		 2841 }
2842
e32eb50d
CK		 2843 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2844 radeon_ring_write(ring,
0f234f5f
AD		 2845#ifdef __BIG_ENDIAN
2846 (2 << 0) |
2847#endif
2848 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2849 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2850 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2851}
2852
bcc1c2a1
AD		 2853
2854static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2855{
fe251e2f
AD		 2856 const __be32 *fw_data;
2857 int i;
2858
2859 if (!rdev->me_fw || !rdev->pfp_fw)
2860 return -EINVAL;
bcc1c2a1 2861
fe251e2f 2862 r700_cp_stop(rdev);
0f234f5f
AD		 2863 WREG32(CP_RB_CNTL,
2864#ifdef __BIG_ENDIAN
2865 BUF_SWAP_32BIT |
2866#endif
2867 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2868
2869 fw_data = (const __be32 *)rdev->pfp_fw->data;
2870 WREG32(CP_PFP_UCODE_ADDR, 0);
2871 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2872 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2873 WREG32(CP_PFP_UCODE_ADDR, 0);
2874
2875 fw_data = (const __be32 *)rdev->me_fw->data;
2876 WREG32(CP_ME_RAM_WADDR, 0);
2877 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2878 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2879
2880 WREG32(CP_PFP_UCODE_ADDR, 0);
2881 WREG32(CP_ME_RAM_WADDR, 0);
2882 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2883 return 0;
2884}
2885
7e7b41d2
AD		 2886static int evergreen_cp_start(struct radeon_device *rdev)
2887{
e32eb50d 2888 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2889 int r, i;
7e7b41d2
AD		 2890 uint32_t cp_me;
2891
e32eb50d 2892 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2893 if (r) {
2894 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2895 return r;
2896 }
e32eb50d
CK		 2897 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2898 radeon_ring_write(ring, 0x1);
2899 radeon_ring_write(ring, 0x0);
2900 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2901 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2902 radeon_ring_write(ring, 0);
2903 radeon_ring_write(ring, 0);
1538a9e0 2904 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 2905
2906 cp_me = 0xff;
2907 WREG32(CP_ME_CNTL, cp_me);
2908
e32eb50d 2909 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2910 if (r) {
2911 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2912 return r;
2913 }
2281a378
AD		 2914
2915 /* setup clear context state */
e32eb50d
CK		 2916 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2917 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2918
2919 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2920 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2921
e32eb50d
CK		 2922 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2923 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2924
2925 /* set clear context state */
e32eb50d
CK		 2926 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2927 radeon_ring_write(ring, 0);
2281a378
AD		 2928
2929 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2930 radeon_ring_write(ring, 0xc0026f00);
2931 radeon_ring_write(ring, 0x00000000);
2932 radeon_ring_write(ring, 0x00000000);
2933 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2934
2935 /* Clear consts */
e32eb50d
CK		 2936 radeon_ring_write(ring, 0xc0036f00);
2937 radeon_ring_write(ring, 0x00000bc4);
2938 radeon_ring_write(ring, 0xffffffff);
2939 radeon_ring_write(ring, 0xffffffff);
2940 radeon_ring_write(ring, 0xffffffff);
2281a378 2941
e32eb50d
CK		 2942 radeon_ring_write(ring, 0xc0026900);
2943 radeon_ring_write(ring, 0x00000316);
2944 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2945 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2946
1538a9e0 2947 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 2948
2949 return 0;
2950}
2951
1109ca09 2952static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2953{
e32eb50d 2954 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2955 u32 tmp;
2956 u32 rb_bufsz;
2957 int r;
2958
2959 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2960 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2961 SOFT_RESET_PA |
2962 SOFT_RESET_SH |
2963 SOFT_RESET_VGT |
a49a50da 2964 SOFT_RESET_SPI |
fe251e2f
AD		 2965 SOFT_RESET_SX));
2966 RREG32(GRBM_SOFT_RESET);
2967 mdelay(15);
2968 WREG32(GRBM_SOFT_RESET, 0);
2969 RREG32(GRBM_SOFT_RESET);
2970
2971 /* Set ring buffer size */
b72a8925
DV		 2972 rb_bufsz = order_base_2(ring->ring_size / 8);
2973 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2974#ifdef __BIG_ENDIAN
2975 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2976#endif
fe251e2f 2977 WREG32(CP_RB_CNTL, tmp);
15d3332f 2978 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2979 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2980
2981 /* Set the write pointer delay */
2982 WREG32(CP_RB_WPTR_DELAY, 0);
2983
2984 /* Initialize the ring buffer's read and write pointers */
2985 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2986 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2987 ring->wptr = 0;
2988 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2989
48fc7f7e 2990 /* set the wb address whether it's enabled or not */
0f234f5f 2991 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2992 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2993 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2994 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2995
2996 if (rdev->wb.enabled)
2997 WREG32(SCRATCH_UMSK, 0xff);
2998 else {
2999 tmp |= RB_NO_UPDATE;
3000 WREG32(SCRATCH_UMSK, 0);
3001 }
3002
fe251e2f
AD		 3003 mdelay(1);
3004 WREG32(CP_RB_CNTL, tmp);
3005
e32eb50d 3006 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 3007 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3008
7e7b41d2 3009 evergreen_cp_start(rdev);
e32eb50d 3010 ring->ready = true;
f712812e 3011 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 3012 if (r) {
e32eb50d 3013 ring->ready = false;
fe251e2f
AD		 3014 return r;
3015 }
3016 return 0;
3017}
bcc1c2a1
AD		 3018
3019/*
3020 * Core functions
3021 */
bcc1c2a1
AD		 3022static void evergreen_gpu_init(struct radeon_device *rdev)
3023{
416a2bd2 3024 u32 gb_addr_config;
32fcdbf4 3025 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 3026 u32 sx_debug_1;
3027 u32 smx_dc_ctl0;
3028 u32 sq_config;
3029 u32 sq_lds_resource_mgmt;
3030 u32 sq_gpr_resource_mgmt_1;
3031 u32 sq_gpr_resource_mgmt_2;
3032 u32 sq_gpr_resource_mgmt_3;
3033 u32 sq_thread_resource_mgmt;
3034 u32 sq_thread_resource_mgmt_2;
3035 u32 sq_stack_resource_mgmt_1;
3036 u32 sq_stack_resource_mgmt_2;
3037 u32 sq_stack_resource_mgmt_3;
3038 u32 vgt_cache_invalidation;
f25a5c63 3039 u32 hdp_host_path_cntl, tmp;
416a2bd2 3040 u32 disabled_rb_mask;
072c44bf 3041 int i, j, ps_thread_count;
32fcdbf4
AD		 3042
3043 switch (rdev->family) {
3044 case CHIP_CYPRESS:
3045 case CHIP_HEMLOCK:
3046 rdev->config.evergreen.num_ses = 2;
3047 rdev->config.evergreen.max_pipes = 4;
3048 rdev->config.evergreen.max_tile_pipes = 8;
3049 rdev->config.evergreen.max_simds = 10;
3050 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3051 rdev->config.evergreen.max_gprs = 256;
3052 rdev->config.evergreen.max_threads = 248;
3053 rdev->config.evergreen.max_gs_threads = 32;
3054 rdev->config.evergreen.max_stack_entries = 512;
3055 rdev->config.evergreen.sx_num_of_sets = 4;
3056 rdev->config.evergreen.sx_max_export_size = 256;
3057 rdev->config.evergreen.sx_max_export_pos_size = 64;
3058 rdev->config.evergreen.sx_max_export_smx_size = 192;
3059 rdev->config.evergreen.max_hw_contexts = 8;
3060 rdev->config.evergreen.sq_num_cf_insts = 2;
3061
3062 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3063 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3064 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3065 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3066 break;
3067 case CHIP_JUNIPER:
3068 rdev->config.evergreen.num_ses = 1;
3069 rdev->config.evergreen.max_pipes = 4;
3070 rdev->config.evergreen.max_tile_pipes = 4;
3071 rdev->config.evergreen.max_simds = 10;
3072 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3073 rdev->config.evergreen.max_gprs = 256;
3074 rdev->config.evergreen.max_threads = 248;
3075 rdev->config.evergreen.max_gs_threads = 32;
3076 rdev->config.evergreen.max_stack_entries = 512;
3077 rdev->config.evergreen.sx_num_of_sets = 4;
3078 rdev->config.evergreen.sx_max_export_size = 256;
3079 rdev->config.evergreen.sx_max_export_pos_size = 64;
3080 rdev->config.evergreen.sx_max_export_smx_size = 192;
3081 rdev->config.evergreen.max_hw_contexts = 8;
3082 rdev->config.evergreen.sq_num_cf_insts = 2;
3083
3084 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3085 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3086 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3087 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3088 break;
3089 case CHIP_REDWOOD:
3090 rdev->config.evergreen.num_ses = 1;
3091 rdev->config.evergreen.max_pipes = 4;
3092 rdev->config.evergreen.max_tile_pipes = 4;
3093 rdev->config.evergreen.max_simds = 5;
3094 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3095 rdev->config.evergreen.max_gprs = 256;
3096 rdev->config.evergreen.max_threads = 248;
3097 rdev->config.evergreen.max_gs_threads = 32;
3098 rdev->config.evergreen.max_stack_entries = 256;
3099 rdev->config.evergreen.sx_num_of_sets = 4;
3100 rdev->config.evergreen.sx_max_export_size = 256;
3101 rdev->config.evergreen.sx_max_export_pos_size = 64;
3102 rdev->config.evergreen.sx_max_export_smx_size = 192;
3103 rdev->config.evergreen.max_hw_contexts = 8;
3104 rdev->config.evergreen.sq_num_cf_insts = 2;
3105
3106 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3107 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3108 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3109 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3110 break;
3111 case CHIP_CEDAR:
3112 default:
3113 rdev->config.evergreen.num_ses = 1;
3114 rdev->config.evergreen.max_pipes = 2;
3115 rdev->config.evergreen.max_tile_pipes = 2;
3116 rdev->config.evergreen.max_simds = 2;
3117 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3118 rdev->config.evergreen.max_gprs = 256;
3119 rdev->config.evergreen.max_threads = 192;
3120 rdev->config.evergreen.max_gs_threads = 16;
3121 rdev->config.evergreen.max_stack_entries = 256;
3122 rdev->config.evergreen.sx_num_of_sets = 4;
3123 rdev->config.evergreen.sx_max_export_size = 128;
3124 rdev->config.evergreen.sx_max_export_pos_size = 32;
3125 rdev->config.evergreen.sx_max_export_smx_size = 96;
3126 rdev->config.evergreen.max_hw_contexts = 4;
3127 rdev->config.evergreen.sq_num_cf_insts = 1;
3128
d5e455e4
AD		 3129 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3130 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3131 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3132 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3133 break;
3134 case CHIP_PALM:
3135 rdev->config.evergreen.num_ses = 1;
3136 rdev->config.evergreen.max_pipes = 2;
3137 rdev->config.evergreen.max_tile_pipes = 2;
3138 rdev->config.evergreen.max_simds = 2;
3139 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3140 rdev->config.evergreen.max_gprs = 256;
3141 rdev->config.evergreen.max_threads = 192;
3142 rdev->config.evergreen.max_gs_threads = 16;
3143 rdev->config.evergreen.max_stack_entries = 256;
3144 rdev->config.evergreen.sx_num_of_sets = 4;
3145 rdev->config.evergreen.sx_max_export_size = 128;
3146 rdev->config.evergreen.sx_max_export_pos_size = 32;
3147 rdev->config.evergreen.sx_max_export_smx_size = 96;
3148 rdev->config.evergreen.max_hw_contexts = 4;
3149 rdev->config.evergreen.sq_num_cf_insts = 1;
3150
d5c5a72f
AD		 3151 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3152 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3153 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3154 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3155 break;
3156 case CHIP_SUMO:
3157 rdev->config.evergreen.num_ses = 1;
3158 rdev->config.evergreen.max_pipes = 4;
bd25f078 3159 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3160 if (rdev->pdev->device == 0x9648)
3161 rdev->config.evergreen.max_simds = 3;
3162 else if ((rdev->pdev->device == 0x9647) ||
3163 (rdev->pdev->device == 0x964a))
3164 rdev->config.evergreen.max_simds = 4;
3165 else
3166 rdev->config.evergreen.max_simds = 5;
3167 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3168 rdev->config.evergreen.max_gprs = 256;
3169 rdev->config.evergreen.max_threads = 248;
3170 rdev->config.evergreen.max_gs_threads = 32;
3171 rdev->config.evergreen.max_stack_entries = 256;
3172 rdev->config.evergreen.sx_num_of_sets = 4;
3173 rdev->config.evergreen.sx_max_export_size = 256;
3174 rdev->config.evergreen.sx_max_export_pos_size = 64;
3175 rdev->config.evergreen.sx_max_export_smx_size = 192;
3176 rdev->config.evergreen.max_hw_contexts = 8;
3177 rdev->config.evergreen.sq_num_cf_insts = 2;
3178
3179 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3180 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3181 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3182 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3183 break;
3184 case CHIP_SUMO2:
3185 rdev->config.evergreen.num_ses = 1;
3186 rdev->config.evergreen.max_pipes = 4;
3187 rdev->config.evergreen.max_tile_pipes = 4;
3188 rdev->config.evergreen.max_simds = 2;
3189 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3190 rdev->config.evergreen.max_gprs = 256;
3191 rdev->config.evergreen.max_threads = 248;
3192 rdev->config.evergreen.max_gs_threads = 32;
3193 rdev->config.evergreen.max_stack_entries = 512;
3194 rdev->config.evergreen.sx_num_of_sets = 4;
3195 rdev->config.evergreen.sx_max_export_size = 256;
3196 rdev->config.evergreen.sx_max_export_pos_size = 64;
3197 rdev->config.evergreen.sx_max_export_smx_size = 192;
50b8f5ae 3198 rdev->config.evergreen.max_hw_contexts = 4;
d5c5a72f
AD		 3199 rdev->config.evergreen.sq_num_cf_insts = 2;
3200
adb68fa2
AD		 3201 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3202 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3203 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3204 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3205 break;
3206 case CHIP_BARTS:
3207 rdev->config.evergreen.num_ses = 2;
3208 rdev->config.evergreen.max_pipes = 4;
3209 rdev->config.evergreen.max_tile_pipes = 8;
3210 rdev->config.evergreen.max_simds = 7;
3211 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3212 rdev->config.evergreen.max_gprs = 256;
3213 rdev->config.evergreen.max_threads = 248;
3214 rdev->config.evergreen.max_gs_threads = 32;
3215 rdev->config.evergreen.max_stack_entries = 512;
3216 rdev->config.evergreen.sx_num_of_sets = 4;
3217 rdev->config.evergreen.sx_max_export_size = 256;
3218 rdev->config.evergreen.sx_max_export_pos_size = 64;
3219 rdev->config.evergreen.sx_max_export_smx_size = 192;
3220 rdev->config.evergreen.max_hw_contexts = 8;
3221 rdev->config.evergreen.sq_num_cf_insts = 2;
3222
3223 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3224 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3225 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3226 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3227 break;
3228 case CHIP_TURKS:
3229 rdev->config.evergreen.num_ses = 1;
3230 rdev->config.evergreen.max_pipes = 4;
3231 rdev->config.evergreen.max_tile_pipes = 4;
3232 rdev->config.evergreen.max_simds = 6;
3233 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3234 rdev->config.evergreen.max_gprs = 256;
3235 rdev->config.evergreen.max_threads = 248;
3236 rdev->config.evergreen.max_gs_threads = 32;
3237 rdev->config.evergreen.max_stack_entries = 256;
3238 rdev->config.evergreen.sx_num_of_sets = 4;
3239 rdev->config.evergreen.sx_max_export_size = 256;
3240 rdev->config.evergreen.sx_max_export_pos_size = 64;
3241 rdev->config.evergreen.sx_max_export_smx_size = 192;
3242 rdev->config.evergreen.max_hw_contexts = 8;
3243 rdev->config.evergreen.sq_num_cf_insts = 2;
3244
3245 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3246 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3247 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3248 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3249 break;
3250 case CHIP_CAICOS:
3251 rdev->config.evergreen.num_ses = 1;
bd25f078 3252 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3253 rdev->config.evergreen.max_tile_pipes = 2;
3254 rdev->config.evergreen.max_simds = 2;
3255 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3256 rdev->config.evergreen.max_gprs = 256;
3257 rdev->config.evergreen.max_threads = 192;
3258 rdev->config.evergreen.max_gs_threads = 16;
3259 rdev->config.evergreen.max_stack_entries = 256;
3260 rdev->config.evergreen.sx_num_of_sets = 4;
3261 rdev->config.evergreen.sx_max_export_size = 128;
3262 rdev->config.evergreen.sx_max_export_pos_size = 32;
3263 rdev->config.evergreen.sx_max_export_smx_size = 96;
3264 rdev->config.evergreen.max_hw_contexts = 4;
3265 rdev->config.evergreen.sq_num_cf_insts = 1;
3266
32fcdbf4
AD		 3267 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3268 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3269 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3270 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3271 break;
3272 }
3273
3274 /* Initialize HDP */
3275 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3276 WREG32((0x2c14 + j), 0x00000000);
3277 WREG32((0x2c18 + j), 0x00000000);
3278 WREG32((0x2c1c + j), 0x00000000);
3279 WREG32((0x2c20 + j), 0x00000000);
3280 WREG32((0x2c24 + j), 0x00000000);
3281 }
3282
3283 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
acc1522a
CK		 3284 WREG32(SRBM_INT_CNTL, 0x1);
3285 WREG32(SRBM_INT_ACK, 0x1);
32fcdbf4 3286
d054ac16
AD		 3287 evergreen_fix_pci_max_read_req_size(rdev);
3288
32fcdbf4 3289 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3290 if ((rdev->family == CHIP_PALM) ||
3291 (rdev->family == CHIP_SUMO) ||
3292 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3293 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3294 else
3295 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3296
1aa52bd3
AD		 3297 /* setup tiling info dword. gb_addr_config is not adequate since it does
3298 * not have bank info, so create a custom tiling dword.
3299 * bits 3:0 num_pipes
3300 * bits 7:4 num_banks
3301 * bits 11:8 group_size
3302 * bits 15:12 row_size
3303 */
3304 rdev->config.evergreen.tile_config = 0;
3305 switch (rdev->config.evergreen.max_tile_pipes) {
3306 case 1:
3307 default:
3308 rdev->config.evergreen.tile_config |= (0 << 0);
3309 break;
3310 case 2:
3311 rdev->config.evergreen.tile_config |= (1 << 0);
3312 break;
3313 case 4:
3314 rdev->config.evergreen.tile_config |= (2 << 0);
3315 break;
3316 case 8:
3317 rdev->config.evergreen.tile_config |= (3 << 0);
3318 break;
3319 }
d698a34d 3320 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3321 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3322 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3323 else {
c8d15edc
AD		 3324 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3325 case 0: /* four banks */
29d65406 3326 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3327 break;
3328 case 1: /* eight banks */
3329 rdev->config.evergreen.tile_config |= 1 << 4;
3330 break;
3331 case 2: /* sixteen banks */
3332 default:
3333 rdev->config.evergreen.tile_config |= 2 << 4;
3334 break;
3335 }
29d65406 3336 }
416a2bd2 3337 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3338 rdev->config.evergreen.tile_config |=
3339 ((gb_addr_config & 0x30000000) >> 28) << 12;
3340
416a2bd2
AD		 3341 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3342 u32 efuse_straps_4;
3343 u32 efuse_straps_3;
32fcdbf4 3344
ff82bbc4
AD		 3345 efuse_straps_4 = RREG32_RCU(0x204);
3346 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3347 tmp = (((efuse_straps_4 & 0xf) << 4) |
3348 ((efuse_straps_3 & 0xf0000000) >> 28));
3349 } else {
3350 tmp = 0;
3351 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3352 u32 rb_disable_bitmap;
3353
3354 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3355 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3356 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3357 tmp <<= 4;
3358 tmp |= rb_disable_bitmap;
32fcdbf4 3359 }
416a2bd2
AD		 3360 }
3361 /* enabled rb are just the one not disabled :) */
3362 disabled_rb_mask = tmp;
cedb655a
AD		 3363 tmp = 0;
3364 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3365 tmp |= (1 << i);
3366 /* if all the backends are disabled, fix it up here */
3367 if ((disabled_rb_mask & tmp) == tmp) {
3368 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3369 disabled_rb_mask &= ~(1 << i);
3370 }
32fcdbf4 3371
65fcf668
AD		 3372 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3373 u32 simd_disable_bitmap;
3374
3375 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3376 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3377 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3378 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3379 tmp <<= 16;
3380 tmp |= simd_disable_bitmap;
3381 }
3382 rdev->config.evergreen.active_simds = hweight32(~tmp);
3383
416a2bd2
AD		 3384 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3385 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3386
416a2bd2
AD		 3387 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3388 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3389 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3390 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3391 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3392 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3393 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3394
f7eb9730
AD		 3395 if ((rdev->config.evergreen.max_backends == 1) &&
3396 (rdev->flags & RADEON_IS_IGP)) {
3397 if ((disabled_rb_mask & 3) == 1) {
3398 /* RB0 disabled, RB1 enabled */
3399 tmp = 0x11111111;
3400 } else {
3401 /* RB1 disabled, RB0 enabled */
3402 tmp = 0x00000000;
3403 }
3404 } else {
3405 tmp = gb_addr_config & NUM_PIPES_MASK;
3406 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3407 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3408 }
416a2bd2 3409 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3410
3411 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3412 WREG32(CGTS_TCC_DISABLE, 0);
3413 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3414 WREG32(CGTS_USER_TCC_DISABLE, 0);
3415
3416 /* set HW defaults for 3D engine */
3417 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3418 ROQ_IB2_START(0x2b)));
3419
3420 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3421
3422 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3423 SYNC_GRADIENT |
3424 SYNC_WALKER |
3425 SYNC_ALIGNER));
3426
3427 sx_debug_1 = RREG32(SX_DEBUG_1);
3428 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3429 WREG32(SX_DEBUG_1, sx_debug_1);
3430
3431
3432 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3433 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3434 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3435 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3436
b866d133
AD		 3437 if (rdev->family <= CHIP_SUMO2)
3438 WREG32(SMX_SAR_CTL0, 0x00010000);
3439
32fcdbf4
AD		 3440 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3441 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3442 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3443
3444 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3445 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3446 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3447
3448 WREG32(VGT_NUM_INSTANCES, 1);
3449 WREG32(SPI_CONFIG_CNTL, 0);
3450 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3451 WREG32(CP_PERFMON_CNTL, 0);
3452
3453 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3454 FETCH_FIFO_HIWATER(0x4) |
3455 DONE_FIFO_HIWATER(0xe0) |
3456 ALU_UPDATE_FIFO_HIWATER(0x8)));
3457
3458 sq_config = RREG32(SQ_CONFIG);
3459 sq_config &= ~(PS_PRIO(3) |
3460 VS_PRIO(3) |
3461 GS_PRIO(3) |
3462 ES_PRIO(3));
3463 sq_config |= (VC_ENABLE |
3464 EXPORT_SRC_C |
3465 PS_PRIO(0) |
3466 VS_PRIO(1) |
3467 GS_PRIO(2) |
3468 ES_PRIO(3));
3469
d5e455e4
AD		 3470 switch (rdev->family) {
3471 case CHIP_CEDAR:
3472 case CHIP_PALM:
d5c5a72f
AD		 3473 case CHIP_SUMO:
3474 case CHIP_SUMO2:
adb68fa2 3475 case CHIP_CAICOS:
32fcdbf4
AD		 3476 /* no vertex cache */
3477 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3478 break;
3479 default:
3480 break;
3481 }
32fcdbf4
AD		 3482
3483 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3484
3485 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3486 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3487 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3488 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3489 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3490 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3491 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3492
d5e455e4
AD		 3493 switch (rdev->family) {
3494 case CHIP_CEDAR:
3495 case CHIP_PALM:
d5c5a72f
AD		 3496 case CHIP_SUMO:
3497 case CHIP_SUMO2:
32fcdbf4 3498 ps_thread_count = 96;
d5e455e4
AD		 3499 break;
3500 default:
32fcdbf4 3501 ps_thread_count = 128;
d5e455e4
AD		 3502 break;
3503 }
32fcdbf4
AD		 3504
3505 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3506 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3507 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3508 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3509 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3510 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3511
3512 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3513 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3514 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3515 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3516 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3517 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3518
3519 WREG32(SQ_CONFIG, sq_config);
3520 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3521 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3522 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3523 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3524 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3525 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3526 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3527 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3528 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3529 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3530
3531 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3532 FORCE_EOV_MAX_REZ_CNT(255)));
3533
d5e455e4
AD		 3534 switch (rdev->family) {
3535 case CHIP_CEDAR:
3536 case CHIP_PALM:
d5c5a72f
AD		 3537 case CHIP_SUMO:
3538 case CHIP_SUMO2:
adb68fa2 3539 case CHIP_CAICOS:
32fcdbf4 3540 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3541 break;
3542 default:
32fcdbf4 3543 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3544 break;
3545 }
32fcdbf4
AD		 3546 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3547 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3548
3549 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3550 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3551 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3552
60a4a3e0
AD		 3553 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3554 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3555
32fcdbf4
AD		 3556 WREG32(CB_PERF_CTR0_SEL_0, 0);
3557 WREG32(CB_PERF_CTR0_SEL_1, 0);
3558 WREG32(CB_PERF_CTR1_SEL_0, 0);
3559 WREG32(CB_PERF_CTR1_SEL_1, 0);
3560 WREG32(CB_PERF_CTR2_SEL_0, 0);
3561 WREG32(CB_PERF_CTR2_SEL_1, 0);
3562 WREG32(CB_PERF_CTR3_SEL_0, 0);
3563 WREG32(CB_PERF_CTR3_SEL_1, 0);
3564
60a4a3e0
AD		 3565 /* clear render buffer base addresses */
3566 WREG32(CB_COLOR0_BASE, 0);
3567 WREG32(CB_COLOR1_BASE, 0);
3568 WREG32(CB_COLOR2_BASE, 0);
3569 WREG32(CB_COLOR3_BASE, 0);
3570 WREG32(CB_COLOR4_BASE, 0);
3571 WREG32(CB_COLOR5_BASE, 0);
3572 WREG32(CB_COLOR6_BASE, 0);
3573 WREG32(CB_COLOR7_BASE, 0);
3574 WREG32(CB_COLOR8_BASE, 0);
3575 WREG32(CB_COLOR9_BASE, 0);
3576 WREG32(CB_COLOR10_BASE, 0);
3577 WREG32(CB_COLOR11_BASE, 0);
3578
3579 /* set the shader const cache sizes to 0 */
3580 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3581 WREG32(i, 0);
3582 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3583 WREG32(i, 0);
3584
f25a5c63
AD		 3585 tmp = RREG32(HDP_MISC_CNTL);
3586 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3587 WREG32(HDP_MISC_CNTL, tmp);
3588
32fcdbf4
AD		 3589 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3590 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3591
3592 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3593
3594 udelay(50);
3595
bcc1c2a1
AD		 3596}
3597
3598int evergreen_mc_init(struct radeon_device *rdev)
3599{
bcc1c2a1
AD		 3600 u32 tmp;
3601 int chansize, numchan;
bcc1c2a1
AD		 3602
3603 /* Get VRAM informations */
3604 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3605 if ((rdev->family == CHIP_PALM) ||
3606 (rdev->family == CHIP_SUMO) ||
3607 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3608 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3609 else
3610 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3611 if (tmp & CHANSIZE_OVERRIDE) {
3612 chansize = 16;
3613 } else if (tmp & CHANSIZE_MASK) {
3614 chansize = 64;
3615 } else {
3616 chansize = 32;
3617 }
3618 tmp = RREG32(MC_SHARED_CHMAP);
3619 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3620 case 0:
3621 default:
3622 numchan = 1;
3623 break;
3624 case 1:
3625 numchan = 2;
3626 break;
3627 case 2:
3628 numchan = 4;
3629 break;
3630 case 3:
3631 numchan = 8;
3632 break;
3633 }
3634 rdev->mc.vram_width = numchan * chansize;
3635 /* Could aper size report 0 ? */
01d73a69
JC		 3636 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3637 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3638 /* Setup GPU memory space */
05b3ef69
AD		 3639 if ((rdev->family == CHIP_PALM) ||
3640 (rdev->family == CHIP_SUMO) ||
3641 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3642 /* size in bytes on fusion */
3643 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3644 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3645 } else {
05b3ef69 3646 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3647 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3648 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3649 }
51e5fcd3 3650 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3651 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3652 radeon_update_bandwidth_info(rdev);
3653
bcc1c2a1
AD		 3654 return 0;
3655}
d594e46a 3656
187e3593 3657void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3658{
64c56e8c 3659 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3660 RREG32(GRBM_STATUS));
64c56e8c 3661 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3662 RREG32(GRBM_STATUS_SE0));
64c56e8c 3663 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3664 RREG32(GRBM_STATUS_SE1));
64c56e8c 3665 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3666 RREG32(SRBM_STATUS));
a65a4369
AD		 3667 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3668 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3669 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3670 RREG32(CP_STALLED_STAT1));
3671 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3672 RREG32(CP_STALLED_STAT2));
3673 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3674 RREG32(CP_BUSY_STAT));
3675 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3676 RREG32(CP_STAT));
eaaa6983
JG		 3677 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3678 RREG32(DMA_STATUS_REG));
168757ea
AD		 3679 if (rdev->family >= CHIP_CAYMAN) {
3680 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3681 RREG32(DMA_STATUS_REG + 0x800));
3682 }
0ecebb9e
AD		 3683}
3684
168757ea 3685bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3686{
a65a4369
AD		 3687 u32 crtc_hung = 0;
3688 u32 crtc_status[6];
3689 u32 i, j, tmp;
3690
3691 for (i = 0; i < rdev->num_crtc; i++) {
3692 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3693 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3694 crtc_hung |= (1 << i);
3695 }
3696 }
3697
3698 for (j = 0; j < 10; j++) {
3699 for (i = 0; i < rdev->num_crtc; i++) {
3700 if (crtc_hung & (1 << i)) {
3701 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3702 if (tmp != crtc_status[i])
3703 crtc_hung &= ~(1 << i);
3704 }
3705 }
3706 if (crtc_hung == 0)
3707 return false;
3708 udelay(100);
3709 }
3710
3711 return true;
3712}
3713
2483b4ea 3714u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
a65a4369
AD		 3715{
3716 u32 reset_mask = 0;
b7630473 3717 u32 tmp;
0ecebb9e 3718
a65a4369
AD		 3719 /* GRBM_STATUS */
3720 tmp = RREG32(GRBM_STATUS);
3721 if (tmp & (PA_BUSY | SC_BUSY |
3722 SH_BUSY | SX_BUSY |
3723 TA_BUSY | VGT_BUSY |
3724 DB_BUSY | CB_BUSY |
3725 SPI_BUSY | VGT_BUSY_NO_DMA))
3726 reset_mask |= RADEON_RESET_GFX;
3727
3728 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3729 CP_BUSY | CP_COHERENCY_BUSY))
3730 reset_mask |= RADEON_RESET_CP;
3731
3732 if (tmp & GRBM_EE_BUSY)
3733 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3734
a65a4369
AD		 3735 /* DMA_STATUS_REG */
3736 tmp = RREG32(DMA_STATUS_REG);
3737 if (!(tmp & DMA_IDLE))
3738 reset_mask |= RADEON_RESET_DMA;
3739
3740 /* SRBM_STATUS2 */
3741 tmp = RREG32(SRBM_STATUS2);
3742 if (tmp & DMA_BUSY)
3743 reset_mask |= RADEON_RESET_DMA;
3744
3745 /* SRBM_STATUS */
3746 tmp = RREG32(SRBM_STATUS);
3747 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3748 reset_mask |= RADEON_RESET_RLC;
3749
3750 if (tmp & IH_BUSY)
3751 reset_mask |= RADEON_RESET_IH;
3752
3753 if (tmp & SEM_BUSY)
3754 reset_mask |= RADEON_RESET_SEM;
3755
3756 if (tmp & GRBM_RQ_PENDING)
3757 reset_mask |= RADEON_RESET_GRBM;
3758
3759 if (tmp & VMC_BUSY)
3760 reset_mask |= RADEON_RESET_VMC;
3761
3762 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3763 MCC_BUSY | MCD_BUSY))
3764 reset_mask |= RADEON_RESET_MC;
3765
3766 if (evergreen_is_display_hung(rdev))
3767 reset_mask |= RADEON_RESET_DISPLAY;
3768
3769 /* VM_L2_STATUS */
3770 tmp = RREG32(VM_L2_STATUS);
3771 if (tmp & L2_BUSY)
3772 reset_mask |= RADEON_RESET_VMC;
3773
d808fc88
AD		 3774 /* Skip MC reset as it's mostly likely not hung, just busy */
3775 if (reset_mask & RADEON_RESET_MC) {
3776 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3777 reset_mask &= ~RADEON_RESET_MC;
3778 }
3779
a65a4369
AD		 3780 return reset_mask;
3781}
3782
3783static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3784{
3785 struct evergreen_mc_save save;
3786 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3787 u32 tmp;
19fc42ed 3788
0ecebb9e 3789 if (reset_mask == 0)
a65a4369 3790 return;
0ecebb9e
AD		 3791
3792 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3793
b7630473
AD		 3794 evergreen_print_gpu_status_regs(rdev);
3795
b7630473
AD		 3796 /* Disable CP parsing/prefetching */
3797 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3798
3799 if (reset_mask & RADEON_RESET_DMA) {
3800 /* Disable DMA */
3801 tmp = RREG32(DMA_RB_CNTL);
3802 tmp &= ~DMA_RB_ENABLE;
3803 WREG32(DMA_RB_CNTL, tmp);
3804 }
3805
b21b6e7a
AD		 3806 udelay(50);
3807
3808 evergreen_mc_stop(rdev, &save);
3809 if (evergreen_mc_wait_for_idle(rdev)) {
3810 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3811 }
3812
b7630473
AD		 3813 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3814 grbm_soft_reset |= SOFT_RESET_DB |
3815 SOFT_RESET_CB |
3816 SOFT_RESET_PA |
3817 SOFT_RESET_SC |
3818 SOFT_RESET_SPI |
3819 SOFT_RESET_SX |
3820 SOFT_RESET_SH |
3821 SOFT_RESET_TC |
3822 SOFT_RESET_TA |
3823 SOFT_RESET_VC |
3824 SOFT_RESET_VGT;
3825 }
3826
3827 if (reset_mask & RADEON_RESET_CP) {
3828 grbm_soft_reset |= SOFT_RESET_CP |
3829 SOFT_RESET_VGT;
3830
3831 srbm_soft_reset |= SOFT_RESET_GRBM;
3832 }
0ecebb9e
AD		 3833
3834 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3835 srbm_soft_reset |= SOFT_RESET_DMA;
3836
a65a4369
AD		 3837 if (reset_mask & RADEON_RESET_DISPLAY)
3838 srbm_soft_reset |= SOFT_RESET_DC;
3839
3840 if (reset_mask & RADEON_RESET_RLC)
3841 srbm_soft_reset |= SOFT_RESET_RLC;
3842
3843 if (reset_mask & RADEON_RESET_SEM)
3844 srbm_soft_reset |= SOFT_RESET_SEM;
3845
3846 if (reset_mask & RADEON_RESET_IH)
3847 srbm_soft_reset |= SOFT_RESET_IH;
3848
3849 if (reset_mask & RADEON_RESET_GRBM)
3850 srbm_soft_reset |= SOFT_RESET_GRBM;
3851
3852 if (reset_mask & RADEON_RESET_VMC)
3853 srbm_soft_reset |= SOFT_RESET_VMC;
3854
24178ec4
AD		 3855 if (!(rdev->flags & RADEON_IS_IGP)) {
3856 if (reset_mask & RADEON_RESET_MC)
3857 srbm_soft_reset |= SOFT_RESET_MC;
3858 }
a65a4369 3859
b7630473
AD		 3860 if (grbm_soft_reset) {
3861 tmp = RREG32(GRBM_SOFT_RESET);
3862 tmp |= grbm_soft_reset;
3863 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3864 WREG32(GRBM_SOFT_RESET, tmp);
3865 tmp = RREG32(GRBM_SOFT_RESET);
3866
3867 udelay(50);
3868
3869 tmp &= ~grbm_soft_reset;
3870 WREG32(GRBM_SOFT_RESET, tmp);
3871 tmp = RREG32(GRBM_SOFT_RESET);
3872 }
3873
3874 if (srbm_soft_reset) {
3875 tmp = RREG32(SRBM_SOFT_RESET);
3876 tmp |= srbm_soft_reset;
3877 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3878 WREG32(SRBM_SOFT_RESET, tmp);
3879 tmp = RREG32(SRBM_SOFT_RESET);
3880
3881 udelay(50);
3882
3883 tmp &= ~srbm_soft_reset;
3884 WREG32(SRBM_SOFT_RESET, tmp);
3885 tmp = RREG32(SRBM_SOFT_RESET);
3886 }
0ecebb9e
AD		 3887
3888 /* Wait a little for things to settle down */
3889 udelay(50);
3890
747943ea 3891 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3892 udelay(50);
3893
b7630473 3894 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3895}
3896
b5470b03
AD		 3897void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3898{
3899 struct evergreen_mc_save save;
3900 u32 tmp, i;
3901
3902 dev_info(rdev->dev, "GPU pci config reset\n");
3903
3904 /* disable dpm? */
3905
3906 /* Disable CP parsing/prefetching */
3907 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3908 udelay(50);
3909 /* Disable DMA */
3910 tmp = RREG32(DMA_RB_CNTL);
3911 tmp &= ~DMA_RB_ENABLE;
3912 WREG32(DMA_RB_CNTL, tmp);
3913 /* XXX other engines? */
3914
3915 /* halt the rlc */
3916 r600_rlc_stop(rdev);
3917
3918 udelay(50);
3919
3920 /* set mclk/sclk to bypass */
3921 rv770_set_clk_bypass_mode(rdev);
3922 /* disable BM */
3923 pci_clear_master(rdev->pdev);
3924 /* disable mem access */
3925 evergreen_mc_stop(rdev, &save);
3926 if (evergreen_mc_wait_for_idle(rdev)) {
3927 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3928 }
3929 /* reset */
3930 radeon_pci_config_reset(rdev);
3931 /* wait for asic to come out of reset */
3932 for (i = 0; i < rdev->usec_timeout; i++) {
3933 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3934 break;
3935 udelay(1);
3936 }
3937}
3938
a2d07b74 3939int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3940{
a65a4369
AD		 3941 u32 reset_mask;
3942
3943 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3944
3945 if (reset_mask)
3946 r600_set_bios_scratch_engine_hung(rdev, true);
3947
b5470b03 3948 /* try soft reset */
a65a4369
AD		 3949 evergreen_gpu_soft_reset(rdev, reset_mask);
3950
3951 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3952
b5470b03
AD		 3953 /* try pci config reset */
3954 if (reset_mask && radeon_hard_reset)
3955 evergreen_gpu_pci_config_reset(rdev);
3956
3957 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3958
a65a4369
AD		 3959 if (!reset_mask)
3960 r600_set_bios_scratch_engine_hung(rdev, false);
3961
3962 return 0;
747943ea
AD		 3963}
3964
123bc183
AD		 3965/**
3966 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3967 *
3968 * @rdev: radeon_device pointer
3969 * @ring: radeon_ring structure holding ring information
3970 *
3971 * Check if the GFX engine is locked up.
3972 * Returns true if the engine appears to be locked up, false if not.
3973 */
3974bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3975{
3976 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3977
3978 if (!(reset_mask & (RADEON_RESET_GFX |
3979 RADEON_RESET_COMPUTE |
3980 RADEON_RESET_CP))) {
ff212f25 3981 radeon_ring_lockup_update(rdev, ring);
123bc183
AD		 3982 return false;
3983 }
123bc183
AD		 3984 return radeon_ring_test_lockup(rdev, ring);
3985}
3986
2948f5e6
AD		 3987/*
3988 * RLC
3989 */
3990#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3991#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3992
3993void sumo_rlc_fini(struct radeon_device *rdev)
3994{
3995 int r;
3996
3997 /* save restore block */
3998 if (rdev->rlc.save_restore_obj) {
3999 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4000 if (unlikely(r != 0))
4001 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4002 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4003 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4004
4005 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4006 rdev->rlc.save_restore_obj = NULL;
4007 }
4008
4009 /* clear state block */
4010 if (rdev->rlc.clear_state_obj) {
4011 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4012 if (unlikely(r != 0))
4013 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4014 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4015 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4016
4017 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4018 rdev->rlc.clear_state_obj = NULL;
4019 }
22c775ce
AD		 4020
4021 /* clear state block */
4022 if (rdev->rlc.cp_table_obj) {
4023 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4024 if (unlikely(r != 0))
4025 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4026 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4027 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4028
4029 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4030 rdev->rlc.cp_table_obj = NULL;
4031 }
2948f5e6
AD		 4032}
4033
22c775ce
AD		 4034#define CP_ME_TABLE_SIZE 96
4035
2948f5e6
AD		 4036int sumo_rlc_init(struct radeon_device *rdev)
4037{
1fd11777 4038 const u32 *src_ptr;
2948f5e6
AD		 4039 volatile u32 *dst_ptr;
4040 u32 dws, data, i, j, k, reg_num;
59a82d0e 4041 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
2948f5e6 4042 u64 reg_list_mc_addr;
1fd11777 4043 const struct cs_section_def *cs_data;
2948f5e6
AD		 4044 int r;
4045
4046 src_ptr = rdev->rlc.reg_list;
4047 dws = rdev->rlc.reg_list_size;
a0f38609
AD		 4048 if (rdev->family >= CHIP_BONAIRE) {
4049 dws += (5 * 16) + 48 + 48 + 64;
4050 }
2948f5e6
AD		 4051 cs_data = rdev->rlc.cs_data;
4052
10b7ca7e
AD		 4053 if (src_ptr) {
4054 /* save restore block */
4055 if (rdev->rlc.save_restore_obj == NULL) {
4056 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4057 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4058 NULL, &rdev->rlc.save_restore_obj);
10b7ca7e
AD		 4059 if (r) {
4060 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4061 return r;
4062 }
4063 }
4064
4065 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4066 if (unlikely(r != 0)) {
4067 sumo_rlc_fini(rdev);
4068 return r;
4069 }
4070 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4071 &rdev->rlc.save_restore_gpu_addr);
2948f5e6 4072 if (r) {
10b7ca7e
AD		 4073 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4074 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4075 sumo_rlc_fini(rdev);
2948f5e6
AD		 4076 return r;
4077 }
2948f5e6 4078
10b7ca7e
AD		 4079 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4080 if (r) {
4081 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4082 sumo_rlc_fini(rdev);
4083 return r;
4084 }
4085 /* write the sr buffer */
4086 dst_ptr = rdev->rlc.sr_ptr;
1fd11777
AD		 4087 if (rdev->family >= CHIP_TAHITI) {
4088 /* SI */
59a82d0e 4089 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6ba81e53 4090 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
1fd11777
AD		 4091 } else {
4092 /* ON/LN/TN */
4093 /* format:
4094 * dw0: (reg2 << 16) | reg1
4095 * dw1: reg1 save space
4096 * dw2: reg2 save space
4097 */
4098 for (i = 0; i < dws; i++) {
4099 data = src_ptr[i] >> 2;
4100 i++;
4101 if (i < dws)
4102 data |= (src_ptr[i] >> 2) << 16;
4103 j = (((i - 1) * 3) / 2);
6ba81e53 4104 dst_ptr[j] = cpu_to_le32(data);
1fd11777
AD		 4105 }
4106 j = ((i * 3) / 2);
6ba81e53 4107 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
10b7ca7e 4108 }
10b7ca7e 4109 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
2948f5e6 4110 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2948f5e6 4111 }
2948f5e6 4112
10b7ca7e
AD		 4113 if (cs_data) {
4114 /* clear state block */
a0f38609
AD		 4115 if (rdev->family >= CHIP_BONAIRE) {
4116 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4117 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4118 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4119 dws = rdev->rlc.clear_state_size + (256 / 4);
4120 } else {
4121 reg_list_num = 0;
4122 dws = 0;
4123 for (i = 0; cs_data[i].section != NULL; i++) {
4124 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4125 reg_list_num++;
4126 dws += cs_data[i].section[j].reg_count;
4127 }
10b7ca7e 4128 }
59a82d0e
AD		 4129 reg_list_blk_index = (3 * reg_list_num + 2);
4130 dws += reg_list_blk_index;
4131 rdev->rlc.clear_state_size = dws;
2948f5e6 4132 }
2948f5e6 4133
10b7ca7e 4134 if (rdev->rlc.clear_state_obj == NULL) {
59a82d0e 4135 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4136 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4137 NULL, &rdev->rlc.clear_state_obj);
10b7ca7e
AD		 4138 if (r) {
4139 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4140 sumo_rlc_fini(rdev);
4141 return r;
4142 }
4143 }
4144 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4145 if (unlikely(r != 0)) {
4146 sumo_rlc_fini(rdev);
4147 return r;
4148 }
4149 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4150 &rdev->rlc.clear_state_gpu_addr);
2948f5e6 4151 if (r) {
10b7ca7e
AD		 4152 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4153 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
2948f5e6
AD		 4154 sumo_rlc_fini(rdev);
4155 return r;
4156 }
2948f5e6 4157
10b7ca7e
AD		 4158 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4159 if (r) {
4160 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4161 sumo_rlc_fini(rdev);
4162 return r;
4163 }
4164 /* set up the cs buffer */
4165 dst_ptr = rdev->rlc.cs_ptr;
a0f38609
AD		 4166 if (rdev->family >= CHIP_BONAIRE) {
4167 cik_get_csb_buffer(rdev, dst_ptr);
4168 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e 4169 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
6ba81e53
AD		 4170 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4171 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4172 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
59a82d0e
AD		 4173 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4174 } else {
4175 reg_list_hdr_blk_index = 0;
4176 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4177 data = upper_32_bits(reg_list_mc_addr);
6ba81e53 4178 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
2948f5e6 4179 reg_list_hdr_blk_index++;
59a82d0e
AD		 4180 for (i = 0; cs_data[i].section != NULL; i++) {
4181 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4182 reg_num = cs_data[i].section[j].reg_count;
4183 data = reg_list_mc_addr & 0xffffffff;
6ba81e53 4184 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4185 reg_list_hdr_blk_index++;
4186
4187 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
6ba81e53 4188 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4189 reg_list_hdr_blk_index++;
4190
4191 data = 0x08000000 | (reg_num * 4);
6ba81e53 4192 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4193 reg_list_hdr_blk_index++;
4194
4195 for (k = 0; k < reg_num; k++) {
4196 data = cs_data[i].section[j].extent[k];
6ba81e53 4197 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
59a82d0e
AD		 4198 }
4199 reg_list_mc_addr += reg_num * 4;
4200 reg_list_blk_index += reg_num;
10b7ca7e 4201 }
2948f5e6 4202 }
6ba81e53 4203 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
2948f5e6 4204 }
10b7ca7e
AD		 4205 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4206 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
2948f5e6 4207 }
2948f5e6 4208
22c775ce
AD		 4209 if (rdev->rlc.cp_table_size) {
4210 if (rdev->rlc.cp_table_obj == NULL) {
02376d82
MD		 4211 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4212 PAGE_SIZE, true,
4213 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4214 NULL, &rdev->rlc.cp_table_obj);
22c775ce
AD		 4215 if (r) {
4216 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4217 sumo_rlc_fini(rdev);
4218 return r;
4219 }
4220 }
4221
4222 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4223 if (unlikely(r != 0)) {
4224 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4225 sumo_rlc_fini(rdev);
4226 return r;
4227 }
4228 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4229 &rdev->rlc.cp_table_gpu_addr);
4230 if (r) {
4231 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4232 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4233 sumo_rlc_fini(rdev);
4234 return r;
4235 }
4236 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4237 if (r) {
4238 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4239 sumo_rlc_fini(rdev);
4240 return r;
4241 }
4242
4243 cik_init_cp_pg_table(rdev);
4244
4245 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4246 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4247
4248 }
2948f5e6
AD		 4249
4250 return 0;
4251}
4252
4253static void evergreen_rlc_start(struct radeon_device *rdev)
4254{
8ba10463
AD		 4255 u32 mask = RLC_ENABLE;
4256
4257 if (rdev->flags & RADEON_IS_IGP) {
4258 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
8ba10463
AD		 4259 }
4260
4261 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4262}
4263
4264int evergreen_rlc_resume(struct radeon_device *rdev)
4265{
4266 u32 i;
4267 const __be32 *fw_data;
4268
4269 if (!rdev->rlc_fw)
4270 return -EINVAL;
4271
4272 r600_rlc_stop(rdev);
4273
4274 WREG32(RLC_HB_CNTL, 0);
4275
4276 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4277 if (rdev->family == CHIP_ARUBA) {
4278 u32 always_on_bitmap =
4279 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4280 /* find out the number of active simds */
4281 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4282 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4283 tmp = hweight32(~tmp);
4284 if (tmp == rdev->config.cayman.max_simds_per_se) {
4285 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4286 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4287 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4288 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4289 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4290 }
4291 } else {
4292 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4293 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4294 }
2948f5e6
AD		 4295 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4296 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4297 } else {
4298 WREG32(RLC_HB_BASE, 0);
4299 WREG32(RLC_HB_RPTR, 0);
4300 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4301 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4302 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4303 }
2948f5e6
AD		 4304 WREG32(RLC_MC_CNTL, 0);
4305 WREG32(RLC_UCODE_CNTL, 0);
4306
4307 fw_data = (const __be32 *)rdev->rlc_fw->data;
4308 if (rdev->family >= CHIP_ARUBA) {
4309 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4310 WREG32(RLC_UCODE_ADDR, i);
4311 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4312 }
4313 } else if (rdev->family >= CHIP_CAYMAN) {
4314 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4315 WREG32(RLC_UCODE_ADDR, i);
4316 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4317 }
4318 } else {
4319 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4320 WREG32(RLC_UCODE_ADDR, i);
4321 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4322 }
4323 }
4324 WREG32(RLC_UCODE_ADDR, 0);
4325
4326 evergreen_rlc_start(rdev);
4327
4328 return 0;
4329}
4330
45f9a39b
AD		 4331/* Interrupts */
4332
4333u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4334{
46437057 4335 if (crtc >= rdev->num_crtc)
45f9a39b 4336 return 0;
46437057
AD		 4337 else
4338 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4339}
4340
4341void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4342{
4343 u32 tmp;
4344
1b37078b
AD		 4345 if (rdev->family >= CHIP_CAYMAN) {
4346 cayman_cp_int_cntl_setup(rdev, 0,
4347 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4348 cayman_cp_int_cntl_setup(rdev, 1, 0);
4349 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4350 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4351 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4352 } else
4353 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4354 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4355 WREG32(DMA_CNTL, tmp);
45f9a39b 4356 WREG32(GRBM_INT_CNTL, 0);
acc1522a 4357 WREG32(SRBM_INT_CNTL, 0);
45f9a39b
AD		 4358 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4359 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4360 if (rdev->num_crtc >= 4) {
18007401
AD		 4361 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4362 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4363 }
4364 if (rdev->num_crtc >= 6) {
18007401
AD		 4365 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4366 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4367 }
45f9a39b
AD		 4368
4369 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4370 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4371 if (rdev->num_crtc >= 4) {
18007401
AD		 4372 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4373 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4374 }
4375 if (rdev->num_crtc >= 6) {
18007401
AD		 4376 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4377 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4378 }
45f9a39b 4379
e9a321c6
AD		 4380 /* only one DAC on DCE5 */
4381 if (!ASIC_IS_DCE5(rdev))
05b3ef69 4382 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4383 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4384
4385 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4386 WREG32(DC_HPD1_INT_CONTROL, tmp);
4387 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4388 WREG32(DC_HPD2_INT_CONTROL, tmp);
4389 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4390 WREG32(DC_HPD3_INT_CONTROL, tmp);
4391 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4392 WREG32(DC_HPD4_INT_CONTROL, tmp);
4393 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4394 WREG32(DC_HPD5_INT_CONTROL, tmp);
4395 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4396 WREG32(DC_HPD6_INT_CONTROL, tmp);
4397
4398}
4399
4400int evergreen_irq_set(struct radeon_device *rdev)
4401{
4402 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4403 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4404 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4405 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4406 u32 grbm_int_cntl = 0;
f122c610 4407 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4408 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4409 u32 thermal_int = 0;
45f9a39b
AD		 4410
4411 if (!rdev->irq.installed) {
fce7d61b 4412 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4413 return -EINVAL;
4414 }
4415 /* don't enable anything if the ih is disabled */
4416 if (!rdev->ih.enabled) {
4417 r600_disable_interrupts(rdev);
4418 /* force the active interrupt state to all disabled */
4419 evergreen_disable_interrupt_state(rdev);
4420 return 0;
4421 }
4422
2bc67b4d
DA		 4423 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4424 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4425 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4426 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4427 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4428 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
d70229f7
AD		 4429 if (rdev->family == CHIP_ARUBA)
4430 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4431 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4432 else
4433 thermal_int = RREG32(CG_THERMAL_INT) &
4434 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4435
f122c610
AD		 4436 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4437 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4438 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4439 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4440 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4441 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4442
233d1ad5
AD		 4443 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4444
1b37078b
AD		 4445 if (rdev->family >= CHIP_CAYMAN) {
4446 /* enable CP interrupts on all rings */
736fc37f 4447 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4448 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4449 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4450 }
736fc37f 4451 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4452 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4453 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4454 }
736fc37f 4455 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4456 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4457 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4458 }
4459 } else {
736fc37f 4460 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4461 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4462 cp_int_cntl |= RB_INT_ENABLE;
4463 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4464 }
45f9a39b 4465 }
1b37078b 4466
233d1ad5
AD		 4467 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4468 DRM_DEBUG("r600_irq_set: sw int dma\n");
4469 dma_cntl |= TRAP_ENABLE;
4470 }
4471
f60cbd11
AD		 4472 if (rdev->family >= CHIP_CAYMAN) {
4473 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4474 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4475 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4476 dma_cntl1 |= TRAP_ENABLE;
4477 }
4478 }
4479
dc50ba7f
AD		 4480 if (rdev->irq.dpm_thermal) {
4481 DRM_DEBUG("dpm thermal\n");
4482 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4483 }
4484
6f34be50 4485 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4486 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4487 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4488 crtc1 |= VBLANK_INT_MASK;
4489 }
6f34be50 4490 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4491 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4492 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4493 crtc2 |= VBLANK_INT_MASK;
4494 }
6f34be50 4495 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4496 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4497 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4498 crtc3 |= VBLANK_INT_MASK;
4499 }
6f34be50 4500 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4501 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4502 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4503 crtc4 |= VBLANK_INT_MASK;
4504 }
6f34be50 4505 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4506 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4507 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4508 crtc5 |= VBLANK_INT_MASK;
4509 }
6f34be50 4510 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4511 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4512 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4513 crtc6 |= VBLANK_INT_MASK;
4514 }
4515 if (rdev->irq.hpd[0]) {
4516 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
2bc67b4d 4517 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4518 }
4519 if (rdev->irq.hpd[1]) {
4520 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
2bc67b4d 4521 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4522 }
4523 if (rdev->irq.hpd[2]) {
4524 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
2bc67b4d 4525 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4526 }
4527 if (rdev->irq.hpd[3]) {
4528 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
2bc67b4d 4529 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4530 }
4531 if (rdev->irq.hpd[4]) {
4532 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
2bc67b4d 4533 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4534 }
4535 if (rdev->irq.hpd[5]) {
4536 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
2bc67b4d 4537 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b 4538 }
f122c610
AD		 4539 if (rdev->irq.afmt[0]) {
4540 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4541 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4542 }
4543 if (rdev->irq.afmt[1]) {
4544 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4545 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4546 }
4547 if (rdev->irq.afmt[2]) {
4548 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4549 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4550 }
4551 if (rdev->irq.afmt[3]) {
4552 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4553 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4554 }
4555 if (rdev->irq.afmt[4]) {
4556 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4557 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4558 }
4559 if (rdev->irq.afmt[5]) {
4560 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4561 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4562 }
45f9a39b 4563
1b37078b
AD		 4564 if (rdev->family >= CHIP_CAYMAN) {
4565 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4566 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4567 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4568 } else
4569 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4570
4571 WREG32(DMA_CNTL, dma_cntl);
4572
f60cbd11
AD		 4573 if (rdev->family >= CHIP_CAYMAN)
4574 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4575
2031f77c 4576 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4577
4578 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4579 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4580 if (rdev->num_crtc >= 4) {
18007401
AD		 4581 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4582 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4583 }
4584 if (rdev->num_crtc >= 6) {
18007401
AD		 4585 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4586 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4587 }
45f9a39b 4588
f5d636d2
CK		 4589 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4590 GRPH_PFLIP_INT_MASK);
4591 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4592 GRPH_PFLIP_INT_MASK);
b7eff394 4593 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 4594 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4595 GRPH_PFLIP_INT_MASK);
4596 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4597 GRPH_PFLIP_INT_MASK);
b7eff394
AD		 4598 }
4599 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 4600 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4601 GRPH_PFLIP_INT_MASK);
4602 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4603 GRPH_PFLIP_INT_MASK);
b7eff394 4604 }
6f34be50 4605
45f9a39b
AD		 4606 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4607 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4608 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4609 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4610 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4611 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4612 if (rdev->family == CHIP_ARUBA)
4613 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4614 else
4615 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4616
f122c610
AD		 4617 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4618 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4619 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4620 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4621 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4622 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4623
c320bb5f
AD		 4624 /* posting read */
4625 RREG32(SRBM_STATUS);
4626
bcc1c2a1
AD		 4627 return 0;
4628}
4629
cbdd4501 4630static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4631{
4632 u32 tmp;
4633
6f34be50
AD		 4634 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4635 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4636 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4637 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4638 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4639 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4640 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4641 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4642 if (rdev->num_crtc >= 4) {
4643 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4644 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4645 }
4646 if (rdev->num_crtc >= 6) {
4647 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4648 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4649 }
6f34be50 4650
f122c610
AD		 4651 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4652 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4653 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4654 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4655 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4656 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4657
6f34be50
AD		 4658 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4659 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4660 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4661 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4662 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4663 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4664 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4665 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4666 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4667 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4668 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4669 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4670
b7eff394
AD		 4671 if (rdev->num_crtc >= 4) {
4672 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4673 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4674 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4675 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4676 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4677 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4678 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4679 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4680 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4681 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4682 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4683 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4684 }
4685
4686 if (rdev->num_crtc >= 6) {
4687 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4688 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4689 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4690 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4691 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4692 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4693 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4694 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4695 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4696 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4697 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4698 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4699 }
45f9a39b 4700
6f34be50 4701 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4702 tmp = RREG32(DC_HPD1_INT_CONTROL);
4703 tmp |= DC_HPDx_INT_ACK;
4704 WREG32(DC_HPD1_INT_CONTROL, tmp);
4705 }
6f34be50 4706 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4707 tmp = RREG32(DC_HPD2_INT_CONTROL);
4708 tmp |= DC_HPDx_INT_ACK;
4709 WREG32(DC_HPD2_INT_CONTROL, tmp);
4710 }
6f34be50 4711 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4712 tmp = RREG32(DC_HPD3_INT_CONTROL);
4713 tmp |= DC_HPDx_INT_ACK;
4714 WREG32(DC_HPD3_INT_CONTROL, tmp);
4715 }
6f34be50 4716 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4717 tmp = RREG32(DC_HPD4_INT_CONTROL);
4718 tmp |= DC_HPDx_INT_ACK;
4719 WREG32(DC_HPD4_INT_CONTROL, tmp);
4720 }
6f34be50 4721 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4722 tmp = RREG32(DC_HPD5_INT_CONTROL);
4723 tmp |= DC_HPDx_INT_ACK;
4724 WREG32(DC_HPD5_INT_CONTROL, tmp);
4725 }
6f34be50 4726 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4727 tmp = RREG32(DC_HPD5_INT_CONTROL);
4728 tmp |= DC_HPDx_INT_ACK;
4729 WREG32(DC_HPD6_INT_CONTROL, tmp);
4730 }
2bc67b4d
DA		 4731
4732 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
4733 tmp = RREG32(DC_HPD1_INT_CONTROL);
4734 tmp |= DC_HPDx_RX_INT_ACK;
4735 WREG32(DC_HPD1_INT_CONTROL, tmp);
4736 }
4737 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
4738 tmp = RREG32(DC_HPD2_INT_CONTROL);
4739 tmp |= DC_HPDx_RX_INT_ACK;
4740 WREG32(DC_HPD2_INT_CONTROL, tmp);
4741 }
4742 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
4743 tmp = RREG32(DC_HPD3_INT_CONTROL);
4744 tmp |= DC_HPDx_RX_INT_ACK;
4745 WREG32(DC_HPD3_INT_CONTROL, tmp);
4746 }
4747 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
4748 tmp = RREG32(DC_HPD4_INT_CONTROL);
4749 tmp |= DC_HPDx_RX_INT_ACK;
4750 WREG32(DC_HPD4_INT_CONTROL, tmp);
4751 }
4752 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
4753 tmp = RREG32(DC_HPD5_INT_CONTROL);
4754 tmp |= DC_HPDx_RX_INT_ACK;
4755 WREG32(DC_HPD5_INT_CONTROL, tmp);
4756 }
4757 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
4758 tmp = RREG32(DC_HPD5_INT_CONTROL);
4759 tmp |= DC_HPDx_RX_INT_ACK;
4760 WREG32(DC_HPD6_INT_CONTROL, tmp);
4761 }
4762
f122c610
AD		 4763 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4764 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4765 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4766 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4767 }
4768 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4769 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4770 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4771 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4772 }
4773 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4774 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4775 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4776 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4777 }
4778 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4779 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4780 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4781 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4782 }
4783 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4784 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4785 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4786 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4787 }
4788 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4789 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4790 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4791 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4792 }
45f9a39b
AD		 4793}
4794
1109ca09 4795static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4796{
45f9a39b
AD		 4797 r600_disable_interrupts(rdev);
4798 /* Wait and acknowledge irq */
4799 mdelay(1);
6f34be50 4800 evergreen_irq_ack(rdev);
45f9a39b
AD		 4801 evergreen_disable_interrupt_state(rdev);
4802}
4803
755d819e 4804void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4805{
4806 evergreen_irq_disable(rdev);
4807 r600_rlc_stop(rdev);
4808}
4809
cbdd4501 4810static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4811{
4812 u32 wptr, tmp;
4813
724c80e1 4814 if (rdev->wb.enabled)
204ae24d 4815 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4816 else
4817 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4818
4819 if (wptr & RB_OVERFLOW) {
11bab0ae 4820 wptr &= ~RB_OVERFLOW;
45f9a39b
AD		 4821 /* When a ring buffer overflow happen start parsing interrupt
4822 * from the last not overwritten vector (wptr + 16). Hopefully
4823 * this should allow us to catchup.
4824 */
6cc2fda2
MD		 4825 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4826 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
45f9a39b
AD		 4827 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4828 tmp = RREG32(IH_RB_CNTL);
4829 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4830 WREG32(IH_RB_CNTL, tmp);
4831 }
4832 return (wptr & rdev->ih.ptr_mask);
4833}
4834
4835int evergreen_irq_process(struct radeon_device *rdev)
4836{
682f1a54
DA		 4837 u32 wptr;
4838 u32 rptr;
45f9a39b
AD		 4839 u32 src_id, src_data;
4840 u32 ring_index;
45f9a39b 4841 bool queue_hotplug = false;
f122c610 4842 bool queue_hdmi = false;
2bc67b4d 4843 bool queue_dp = false;
dc50ba7f 4844 bool queue_thermal = false;
54e2e49c 4845 u32 status, addr;
45f9a39b 4846
682f1a54 4847 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4848 return IRQ_NONE;
4849
682f1a54 4850 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4851
4852restart_ih:
4853 /* is somebody else already processing irqs? */
4854 if (atomic_xchg(&rdev->ih.lock, 1))
4855 return IRQ_NONE;
4856
682f1a54
DA		 4857 rptr = rdev->ih.rptr;
4858 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4859
964f6645
BH		 4860 /* Order reading of wptr vs. reading of IH ring data */
4861 rmb();
4862
45f9a39b 4863 /* display interrupts */
6f34be50 4864 evergreen_irq_ack(rdev);
45f9a39b 4865
45f9a39b
AD		 4866 while (rptr != wptr) {
4867 /* wptr/rptr are in bytes! */
4868 ring_index = rptr / 4;
0f234f5f
AD		 4869 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4870 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4871
4872 switch (src_id) {
4873 case 1: /* D1 vblank/vline */
4874 switch (src_data) {
4875 case 0: /* D1 vblank */
6f34be50 4876 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4877 if (rdev->irq.crtc_vblank_int[0]) {
4878 drm_handle_vblank(rdev->ddev, 0);
4879 rdev->pm.vblank_sync = true;
4880 wake_up(&rdev->irq.vblank_queue);
4881 }
736fc37f 4882 if (atomic_read(&rdev->irq.pflip[0]))
1a0e7918 4883 radeon_crtc_handle_vblank(rdev, 0);
6f34be50 4884 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4885 DRM_DEBUG("IH: D1 vblank\n");
4886 }
4887 break;
4888 case 1: /* D1 vline */
6f34be50
AD		 4889 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4890 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4891 DRM_DEBUG("IH: D1 vline\n");
4892 }
4893 break;
4894 default:
4895 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4896 break;
4897 }
4898 break;
4899 case 2: /* D2 vblank/vline */
4900 switch (src_data) {
4901 case 0: /* D2 vblank */
6f34be50 4902 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4903 if (rdev->irq.crtc_vblank_int[1]) {
4904 drm_handle_vblank(rdev->ddev, 1);
4905 rdev->pm.vblank_sync = true;
4906 wake_up(&rdev->irq.vblank_queue);
4907 }
736fc37f 4908 if (atomic_read(&rdev->irq.pflip[1]))
1a0e7918 4909 radeon_crtc_handle_vblank(rdev, 1);
6f34be50 4910 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4911 DRM_DEBUG("IH: D2 vblank\n");
4912 }
4913 break;
4914 case 1: /* D2 vline */
6f34be50
AD		 4915 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4916 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4917 DRM_DEBUG("IH: D2 vline\n");
4918 }
4919 break;
4920 default:
4921 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4922 break;
4923 }
4924 break;
4925 case 3: /* D3 vblank/vline */
4926 switch (src_data) {
4927 case 0: /* D3 vblank */
6f34be50
AD		 4928 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4929 if (rdev->irq.crtc_vblank_int[2]) {
4930 drm_handle_vblank(rdev->ddev, 2);
4931 rdev->pm.vblank_sync = true;
4932 wake_up(&rdev->irq.vblank_queue);
4933 }
736fc37f 4934 if (atomic_read(&rdev->irq.pflip[2]))
1a0e7918 4935 radeon_crtc_handle_vblank(rdev, 2);
6f34be50 4936 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4937 DRM_DEBUG("IH: D3 vblank\n");
4938 }
4939 break;
4940 case 1: /* D3 vline */
6f34be50
AD		 4941 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4942 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4943 DRM_DEBUG("IH: D3 vline\n");
4944 }
4945 break;
4946 default:
4947 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4948 break;
4949 }
4950 break;
4951 case 4: /* D4 vblank/vline */
4952 switch (src_data) {
4953 case 0: /* D4 vblank */
6f34be50
AD		 4954 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4955 if (rdev->irq.crtc_vblank_int[3]) {
4956 drm_handle_vblank(rdev->ddev, 3);
4957 rdev->pm.vblank_sync = true;
4958 wake_up(&rdev->irq.vblank_queue);
4959 }
736fc37f 4960 if (atomic_read(&rdev->irq.pflip[3]))
1a0e7918 4961 radeon_crtc_handle_vblank(rdev, 3);
6f34be50 4962 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4963 DRM_DEBUG("IH: D4 vblank\n");
4964 }
4965 break;
4966 case 1: /* D4 vline */
6f34be50
AD		 4967 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4968 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4969 DRM_DEBUG("IH: D4 vline\n");
4970 }
4971 break;
4972 default:
4973 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4974 break;
4975 }
4976 break;
4977 case 5: /* D5 vblank/vline */
4978 switch (src_data) {
4979 case 0: /* D5 vblank */
6f34be50
AD		 4980 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4981 if (rdev->irq.crtc_vblank_int[4]) {
4982 drm_handle_vblank(rdev->ddev, 4);
4983 rdev->pm.vblank_sync = true;
4984 wake_up(&rdev->irq.vblank_queue);
4985 }
736fc37f 4986 if (atomic_read(&rdev->irq.pflip[4]))
1a0e7918 4987 radeon_crtc_handle_vblank(rdev, 4);
6f34be50 4988 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4989 DRM_DEBUG("IH: D5 vblank\n");
4990 }
4991 break;
4992 case 1: /* D5 vline */
6f34be50
AD		 4993 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4994 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4995 DRM_DEBUG("IH: D5 vline\n");
4996 }
4997 break;
4998 default:
4999 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5000 break;
5001 }
5002 break;
5003 case 6: /* D6 vblank/vline */
5004 switch (src_data) {
5005 case 0: /* D6 vblank */
6f34be50
AD		 5006 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
5007 if (rdev->irq.crtc_vblank_int[5]) {
5008 drm_handle_vblank(rdev->ddev, 5);
5009 rdev->pm.vblank_sync = true;
5010 wake_up(&rdev->irq.vblank_queue);
5011 }
736fc37f 5012 if (atomic_read(&rdev->irq.pflip[5]))
1a0e7918 5013 radeon_crtc_handle_vblank(rdev, 5);
6f34be50 5014 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 5015 DRM_DEBUG("IH: D6 vblank\n");
5016 }
5017 break;
5018 case 1: /* D6 vline */
6f34be50
AD		 5019 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
5020 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 5021 DRM_DEBUG("IH: D6 vline\n");
5022 }
5023 break;
5024 default:
5025 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5026 break;
5027 }
5028 break;
f5d636d2
CK		 5029 case 8: /* D1 page flip */
5030 case 10: /* D2 page flip */
5031 case 12: /* D3 page flip */
5032 case 14: /* D4 page flip */
5033 case 16: /* D5 page flip */
5034 case 18: /* D6 page flip */
5035 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 5036 if (radeon_use_pflipirq > 0)
5037 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 5038 break;
45f9a39b
AD		 5039 case 42: /* HPD hotplug */
5040 switch (src_data) {
5041 case 0:
6f34be50
AD		 5042 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5043 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 5044 queue_hotplug = true;
5045 DRM_DEBUG("IH: HPD1\n");
5046 }
5047 break;
5048 case 1:
6f34be50
AD		 5049 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5050 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 5051 queue_hotplug = true;
5052 DRM_DEBUG("IH: HPD2\n");
5053 }
5054 break;
5055 case 2:
6f34be50
AD		 5056 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5057 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 5058 queue_hotplug = true;
5059 DRM_DEBUG("IH: HPD3\n");
5060 }
5061 break;
5062 case 3:
6f34be50
AD		 5063 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5064 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 5065 queue_hotplug = true;
5066 DRM_DEBUG("IH: HPD4\n");
5067 }
5068 break;
5069 case 4:
6f34be50
AD		 5070 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5071 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 5072 queue_hotplug = true;
5073 DRM_DEBUG("IH: HPD5\n");
5074 }
5075 break;
5076 case 5:
6f34be50
AD		 5077 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5078 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 5079 queue_hotplug = true;
5080 DRM_DEBUG("IH: HPD6\n");
5081 }
5082 break;
2bc67b4d
DA		 5083 case 6:
5084 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
5085 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
5086 queue_dp = true;
5087 DRM_DEBUG("IH: HPD_RX 1\n");
5088 }
5089 break;
5090 case 7:
5091 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
5092 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
5093 queue_dp = true;
5094 DRM_DEBUG("IH: HPD_RX 2\n");
5095 }
5096 break;
5097 case 8:
5098 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
5099 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
5100 queue_dp = true;
5101 DRM_DEBUG("IH: HPD_RX 3\n");
5102 }
5103 break;
5104 case 9:
5105 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
5106 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
5107 queue_dp = true;
5108 DRM_DEBUG("IH: HPD_RX 4\n");
5109 }
5110 break;
5111 case 10:
5112 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
5113 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
5114 queue_dp = true;
5115 DRM_DEBUG("IH: HPD_RX 5\n");
5116 }
5117 break;
5118 case 11:
5119 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
5120 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
5121 queue_dp = true;
5122 DRM_DEBUG("IH: HPD_RX 6\n");
5123 }
5124 break;
45f9a39b
AD		 5125 default:
5126 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5127 break;
5128 }
5129 break;
f122c610
AD		 5130 case 44: /* hdmi */
5131 switch (src_data) {
5132 case 0:
5133 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
5134 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5135 queue_hdmi = true;
5136 DRM_DEBUG("IH: HDMI0\n");
5137 }
5138 break;
5139 case 1:
5140 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
5141 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5142 queue_hdmi = true;
5143 DRM_DEBUG("IH: HDMI1\n");
5144 }
5145 break;
5146 case 2:
5147 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
5148 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5149 queue_hdmi = true;
5150 DRM_DEBUG("IH: HDMI2\n");
5151 }
5152 break;
5153 case 3:
5154 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
5155 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5156 queue_hdmi = true;
5157 DRM_DEBUG("IH: HDMI3\n");
5158 }
5159 break;
5160 case 4:
5161 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
5162 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5163 queue_hdmi = true;
5164 DRM_DEBUG("IH: HDMI4\n");
5165 }
5166 break;
5167 case 5:
5168 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5169 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5170 queue_hdmi = true;
5171 DRM_DEBUG("IH: HDMI5\n");
5172 }
5173 break;
5174 default:
5175 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5176 break;
5177 }
acc1522a
CK		 5178 case 96:
5179 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5180 WREG32(SRBM_INT_ACK, 0x1);
5181 break;
f2ba57b5
CK		 5182 case 124: /* UVD */
5183 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5184 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 5185 break;
ae133a11
CK		 5186 case 146:
5187 case 147:
54e2e49c
AD		 5188 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5189 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
9b7d786b
CK		 5190 /* reset addr and status */
5191 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5192 if (addr == 0x0 && status == 0x0)
5193 break;
ae133a11
CK		 5194 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5195 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
54e2e49c 5196 addr);
ae133a11 5197 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
54e2e49c
AD		 5198 status);
5199 cayman_vm_decode_fault(rdev, status, addr);
ae133a11 5200 break;
45f9a39b
AD		 5201 case 176: /* CP_INT in ring buffer */
5202 case 177: /* CP_INT in IB1 */
5203 case 178: /* CP_INT in IB2 */
5204 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 5205 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 5206 break;
5207 case 181: /* CP EOP event */
5208 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 5209 if (rdev->family >= CHIP_CAYMAN) {
5210 switch (src_data) {
5211 case 0:
5212 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5213 break;
5214 case 1:
5215 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5216 break;
5217 case 2:
5218 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5219 break;
5220 }
5221 } else
5222 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 5223 break;
233d1ad5
AD		 5224 case 224: /* DMA trap event */
5225 DRM_DEBUG("IH: DMA trap\n");
5226 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5227 break;
dc50ba7f
AD		 5228 case 230: /* thermal low to high */
5229 DRM_DEBUG("IH: thermal low to high\n");
5230 rdev->pm.dpm.thermal.high_to_low = false;
5231 queue_thermal = true;
5232 break;
5233 case 231: /* thermal high to low */
5234 DRM_DEBUG("IH: thermal high to low\n");
5235 rdev->pm.dpm.thermal.high_to_low = true;
5236 queue_thermal = true;
5237 break;
2031f77c 5238 case 233: /* GUI IDLE */
303c805c 5239 DRM_DEBUG("IH: GUI idle\n");
2031f77c 5240 break;
f60cbd11
AD		 5241 case 244: /* DMA trap event */
5242 if (rdev->family >= CHIP_CAYMAN) {
5243 DRM_DEBUG("IH: DMA1 trap\n");
5244 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5245 }
5246 break;
45f9a39b
AD		 5247 default:
5248 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5249 break;
5250 }
5251
5252 /* wptr/rptr are in bytes! */
5253 rptr += 16;
5254 rptr &= rdev->ih.ptr_mask;
f55e03b9 5255 WREG32(IH_RB_RPTR, rptr);
45f9a39b 5256 }
2bc67b4d
DA		 5257 if (queue_dp)
5258 schedule_work(&rdev->dp_work);
45f9a39b 5259 if (queue_hotplug)
32c87fca 5260 schedule_work(&rdev->hotplug_work);
f122c610
AD		 5261 if (queue_hdmi)
5262 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 5263 if (queue_thermal && rdev->pm.dpm_enabled)
5264 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b 5265 rdev->ih.rptr = rptr;
c20dc369
CK		 5266 atomic_set(&rdev->ih.lock, 0);
5267
5268 /* make sure wptr hasn't changed while processing */
5269 wptr = evergreen_get_ih_wptr(rdev);
5270 if (wptr != rptr)
5271 goto restart_ih;
5272
45f9a39b
AD		 5273 return IRQ_HANDLED;
5274}
5275
bcc1c2a1
AD		 5276static int evergreen_startup(struct radeon_device *rdev)
5277{
f2ba57b5 5278 struct radeon_ring *ring;
bcc1c2a1
AD		 5279 int r;
5280
9e46a48d 5281 /* enable pcie gen2 link */
cd54033a 5282 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5283 /* enable aspm */
5284 evergreen_program_aspm(rdev);
9e46a48d 5285
e5903d39
AD		 5286 /* scratch needs to be initialized before MC */
5287 r = r600_vram_scratch_init(rdev);
5288 if (r)
5289 return r;
5290
6fab3feb
AD		 5291 evergreen_mc_program(rdev);
5292
6c7bccea 5293 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
755d819e 5294 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5295 if (r) {
0af62b01 5296 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5297 return r;
5298 }
5299 }
fe251e2f 5300
bcc1c2a1 5301 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5302 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5303 } else {
5304 r = evergreen_pcie_gart_enable(rdev);
5305 if (r)
5306 return r;
5307 }
bcc1c2a1 5308 evergreen_gpu_init(rdev);
bcc1c2a1 5309
2948f5e6
AD		 5310 /* allocate rlc buffers */
5311 if (rdev->flags & RADEON_IS_IGP) {
5312 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
1fd11777
AD		 5313 rdev->rlc.reg_list_size =
5314 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
2948f5e6
AD		 5315 rdev->rlc.cs_data = evergreen_cs_data;
5316 r = sumo_rlc_init(rdev);
5317 if (r) {
5318 DRM_ERROR("Failed to init rlc BOs!\n");
5319 return r;
5320 }
5321 }
5322
724c80e1
AD		 5323 /* allocate wb buffer */
5324 r = radeon_wb_init(rdev);
5325 if (r)
5326 return r;
5327
30eb77f4
JG		 5328 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5329 if (r) {
5330 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5331 return r;
5332 }
5333
233d1ad5
AD		 5334 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5335 if (r) {
5336 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5337 return r;
5338 }
5339
e409b128 5340 r = uvd_v2_2_resume(rdev);
f2ba57b5
CK		 5341 if (!r) {
5342 r = radeon_fence_driver_start_ring(rdev,
5343 R600_RING_TYPE_UVD_INDEX);
5344 if (r)
5345 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5346 }
5347
5348 if (r)
5349 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5350
bcc1c2a1 5351 /* Enable IRQ */
e49f3959
AH		 5352 if (!rdev->irq.installed) {
5353 r = radeon_irq_kms_init(rdev);
5354 if (r)
5355 return r;
5356 }
5357
bcc1c2a1
AD		 5358 r = r600_irq_init(rdev);
5359 if (r) {
5360 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5361 radeon_irq_kms_fini(rdev);
5362 return r;
5363 }
45f9a39b 5364 evergreen_irq_set(rdev);
bcc1c2a1 5365
f2ba57b5 5366 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5367 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 5368 RADEON_CP_PACKET2);
bcc1c2a1
AD		 5369 if (r)
5370 return r;
233d1ad5
AD		 5371
5372 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5373 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 5374 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5375 if (r)
5376 return r;
5377
bcc1c2a1
AD		 5378 r = evergreen_cp_load_microcode(rdev);
5379 if (r)
5380 return r;
fe251e2f 5381 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5382 if (r)
5383 return r;
5384 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5385 if (r)
5386 return r;
fe251e2f 5387
f2ba57b5
CK		 5388 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5389 if (ring->ring_size) {
02c9f7fa 5390 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 5391 RADEON_CP_PACKET2);
f2ba57b5 5392 if (!r)
e409b128 5393 r = uvd_v1_0_init(rdev);
f2ba57b5
CK		 5394
5395 if (r)
5396 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5397 }
5398
2898c348
CK		 5399 r = radeon_ib_pool_init(rdev);
5400 if (r) {
5401 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5402 return r;
2898c348 5403 }
b15ba512 5404
bfc1f97d 5405 r = radeon_audio_init(rdev);
69d2ae57
RM		 5406 if (r) {
5407 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5408 return r;
5409 }
5410
bcc1c2a1
AD		 5411 return 0;
5412}
5413
5414int evergreen_resume(struct radeon_device *rdev)
5415{
5416 int r;
5417
86f5c9ed
AD		 5418 /* reset the asic, the gfx blocks are often in a bad state
5419 * after the driver is unloaded or after a resume
5420 */
5421 if (radeon_asic_reset(rdev))
5422 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5423 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5424 * posting will perform necessary task to bring back GPU into good
5425 * shape.
5426 */
5427 /* post card */
5428 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5429
d4788db3
AD		 5430 /* init golden registers */
5431 evergreen_init_golden_registers(rdev);
5432
bc6a6295
AD		 5433 if (rdev->pm.pm_method == PM_METHOD_DPM)
5434 radeon_pm_resume(rdev);
6c7bccea 5435
b15ba512 5436 rdev->accel_working = true;
bcc1c2a1
AD		 5437 r = evergreen_startup(rdev);
5438 if (r) {
755d819e 5439 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5440 rdev->accel_working = false;
bcc1c2a1
AD		 5441 return r;
5442 }
fe251e2f 5443
bcc1c2a1
AD		 5444 return r;
5445
5446}
5447
5448int evergreen_suspend(struct radeon_device *rdev)
5449{
6c7bccea 5450 radeon_pm_suspend(rdev);
7991d665 5451 radeon_audio_fini(rdev);
e409b128 5452 uvd_v1_0_fini(rdev);
f2ba57b5 5453 radeon_uvd_suspend(rdev);
bcc1c2a1 5454 r700_cp_stop(rdev);
233d1ad5 5455 r600_dma_stop(rdev);
45f9a39b 5456 evergreen_irq_suspend(rdev);
724c80e1 5457 radeon_wb_disable(rdev);
bcc1c2a1 5458 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5459
5460 return 0;
5461}
5462
bcc1c2a1
AD		 5463/* Plan is to move initialization in that function and use
5464 * helper function so that radeon_device_init pretty much
5465 * do nothing more than calling asic specific function. This
5466 * should also allow to remove a bunch of callback function
5467 * like vram_info.
5468 */
5469int evergreen_init(struct radeon_device *rdev)
5470{
5471 int r;
5472
bcc1c2a1
AD		 5473 /* Read BIOS */
5474 if (!radeon_get_bios(rdev)) {
5475 if (ASIC_IS_AVIVO(rdev))
5476 return -EINVAL;
5477 }
5478 /* Must be an ATOMBIOS */
5479 if (!rdev->is_atom_bios) {
755d819e 5480 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5481 return -EINVAL;
5482 }
5483 r = radeon_atombios_init(rdev);
5484 if (r)
5485 return r;
86f5c9ed
AD		 5486 /* reset the asic, the gfx blocks are often in a bad state
5487 * after the driver is unloaded or after a resume
5488 */
5489 if (radeon_asic_reset(rdev))
5490 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5491 /* Post card if necessary */
fd909c37 5492 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5493 if (!rdev->bios) {
5494 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5495 return -EINVAL;
5496 }
5497 DRM_INFO("GPU not posted. posting now...\n");
5498 atom_asic_init(rdev->mode_info.atom_context);
5499 }
d4788db3
AD		 5500 /* init golden registers */
5501 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5502 /* Initialize scratch registers */
5503 r600_scratch_init(rdev);
5504 /* Initialize surface registers */
5505 radeon_surface_init(rdev);
5506 /* Initialize clocks */
5507 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5508 /* Fence driver */
5509 r = radeon_fence_driver_init(rdev);
5510 if (r)
5511 return r;
d594e46a
JG		 5512 /* initialize AGP */
5513 if (rdev->flags & RADEON_IS_AGP) {
5514 r = radeon_agp_init(rdev);
5515 if (r)
5516 radeon_agp_disable(rdev);
5517 }
5518 /* initialize memory controller */
bcc1c2a1
AD		 5519 r = evergreen_mc_init(rdev);
5520 if (r)
5521 return r;
5522 /* Memory manager */
5523 r = radeon_bo_init(rdev);
5524 if (r)
5525 return r;
45f9a39b 5526
01ac8794
AD		 5527 if (ASIC_IS_DCE5(rdev)) {
5528 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5529 r = ni_init_microcode(rdev);
5530 if (r) {
5531 DRM_ERROR("Failed to load firmware!\n");
5532 return r;
5533 }
5534 }
5535 } else {
5536 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5537 r = r600_init_microcode(rdev);
5538 if (r) {
5539 DRM_ERROR("Failed to load firmware!\n");
5540 return r;
5541 }
5542 }
5543 }
5544
6c7bccea
AD		 5545 /* Initialize power management */
5546 radeon_pm_init(rdev);
5547
e32eb50d
CK		 5548 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5549 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5550
233d1ad5
AD		 5551 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5552 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5553
f2ba57b5
CK		 5554 r = radeon_uvd_init(rdev);
5555 if (!r) {
5556 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5557 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5558 4096);
5559 }
5560
bcc1c2a1
AD		 5561 rdev->ih.ring_obj = NULL;
5562 r600_ih_ring_init(rdev, 64 * 1024);
5563
5564 r = r600_pcie_gart_init(rdev);
5565 if (r)
5566 return r;
0fcdb61e 5567
148a03bc 5568 rdev->accel_working = true;
bcc1c2a1
AD		 5569 r = evergreen_startup(rdev);
5570 if (r) {
fe251e2f
AD		 5571 dev_err(rdev->dev, "disabling GPU acceleration\n");
5572 r700_cp_fini(rdev);
233d1ad5 5573 r600_dma_fini(rdev);
fe251e2f 5574 r600_irq_fini(rdev);
2948f5e6
AD		 5575 if (rdev->flags & RADEON_IS_IGP)
5576 sumo_rlc_fini(rdev);
724c80e1 5577 radeon_wb_fini(rdev);
2898c348 5578 radeon_ib_pool_fini(rdev);
fe251e2f 5579 radeon_irq_kms_fini(rdev);
0fcdb61e 5580 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5581 rdev->accel_working = false;
5582 }
77e00f2e
AD		 5583
5584 /* Don't start up if the MC ucode is missing on BTC parts.
5585 * The default clocks and voltages before the MC ucode
5586 * is loaded are not suffient for advanced operations.
5587 */
5588 if (ASIC_IS_DCE5(rdev)) {
5589 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5590 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5591 return -EINVAL;
5592 }
5593 }
5594
bcc1c2a1
AD		 5595 return 0;
5596}
5597
5598void evergreen_fini(struct radeon_device *rdev)
5599{
6c7bccea 5600 radeon_pm_fini(rdev);
7991d665 5601 radeon_audio_fini(rdev);
45f9a39b 5602 r700_cp_fini(rdev);
233d1ad5 5603 r600_dma_fini(rdev);
bcc1c2a1 5604 r600_irq_fini(rdev);
2948f5e6
AD		 5605 if (rdev->flags & RADEON_IS_IGP)
5606 sumo_rlc_fini(rdev);
724c80e1 5607 radeon_wb_fini(rdev);
2898c348 5608 radeon_ib_pool_fini(rdev);
bcc1c2a1 5609 radeon_irq_kms_fini(rdev);
e409b128 5610 uvd_v1_0_fini(rdev);
f2ba57b5 5611 radeon_uvd_fini(rdev);
d9654413 5612 evergreen_pcie_gart_fini(rdev);
16cdf04d 5613 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5614 radeon_gem_fini(rdev);
5615 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5616 radeon_agp_fini(rdev);
5617 radeon_bo_fini(rdev);
5618 radeon_atombios_fini(rdev);
5619 kfree(rdev->bios);
5620 rdev->bios = NULL;
bcc1c2a1 5621}
9e46a48d 5622
b07759bf 5623void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5624{
7e0e4196 5625 u32 link_width_cntl, speed_cntl;
9e46a48d 5626
d42dd579
AD		 5627 if (radeon_pcie_gen2 == 0)
5628 return;
5629
9e46a48d
AD		 5630 if (rdev->flags & RADEON_IS_IGP)
5631 return;
5632
5633 if (!(rdev->flags & RADEON_IS_PCIE))
5634 return;
5635
5636 /* x2 cards have a special sequence */
5637 if (ASIC_IS_X2(rdev))
5638 return;
5639
7e0e4196
KSS		 5640 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5641 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5642 return;
5643
492d2b61 5644 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5645 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5646 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5647 return;
5648 }
5649
197bbb3d
DA		 5650 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5651
9e46a48d
AD		 5652 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5653 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5654
492d2b61 5655 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5656 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5657 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5658
492d2b61 5659 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5660 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5661 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5662
492d2b61 5663 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5664 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5665 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5666
492d2b61 5667 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5668 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5669 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5670
492d2b61 5671 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5672 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5673 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5674
5675 } else {
492d2b61 5676 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5677 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5678 if (1)
5679 link_width_cntl |= LC_UPCONFIGURE_DIS;
5680 else
5681 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5682 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5683 }
5684}
f52382d7
AD		 5685
5686void evergreen_program_aspm(struct radeon_device *rdev)
5687{
5688 u32 data, orig;
5689 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5690 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5691 /* fusion_platform = true
5692 * if the system is a fusion system
5693 * (APU or DGPU in a fusion system).
5694 * todo: check if the system is a fusion platform.
5695 */
5696 bool fusion_platform = false;
5697
1294d4a3
AD		 5698 if (radeon_aspm == 0)
5699 return;
5700
f52382d7
AD		 5701 if (!(rdev->flags & RADEON_IS_PCIE))
5702 return;
5703
5704 switch (rdev->family) {
5705 case CHIP_CYPRESS:
5706 case CHIP_HEMLOCK:
5707 case CHIP_JUNIPER:
5708 case CHIP_REDWOOD:
5709 case CHIP_CEDAR:
5710 case CHIP_SUMO:
5711 case CHIP_SUMO2:
5712 case CHIP_PALM:
5713 case CHIP_ARUBA:
5714 disable_l0s = true;
5715 break;
5716 default:
5717 disable_l0s = false;
5718 break;
5719 }
5720
5721 if (rdev->flags & RADEON_IS_IGP)
5722 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5723
5724 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5725 if (fusion_platform)
5726 data &= ~MULTI_PIF;
5727 else
5728 data |= MULTI_PIF;
5729 if (data != orig)
5730 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5731
5732 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5733 if (fusion_platform)
5734 data &= ~MULTI_PIF;
5735 else
5736 data |= MULTI_PIF;
5737 if (data != orig)
5738 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5739
5740 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5741 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5742 if (!disable_l0s) {
5743 if (rdev->family >= CHIP_BARTS)
5744 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5745 else
5746 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5747 }
5748
5749 if (!disable_l1) {
5750 if (rdev->family >= CHIP_BARTS)
5751 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5752 else
5753 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5754
5755 if (!disable_plloff_in_l1) {
5756 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5757 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5758 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5759 if (data != orig)
5760 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5761
5762 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5763 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5764 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5765 if (data != orig)
5766 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5767
5768 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5769 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5770 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5771 if (data != orig)
5772 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5773
5774 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5775 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5776 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5777 if (data != orig)
5778 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5779
5780 if (rdev->family >= CHIP_BARTS) {
5781 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5782 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5783 data |= PLL_RAMP_UP_TIME_0(4);
5784 if (data != orig)
5785 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5786
5787 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5788 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5789 data |= PLL_RAMP_UP_TIME_1(4);
5790 if (data != orig)
5791 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5792
5793 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5794 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5795 data |= PLL_RAMP_UP_TIME_0(4);
5796 if (data != orig)
5797 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5798
5799 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5800 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5801 data |= PLL_RAMP_UP_TIME_1(4);
5802 if (data != orig)
5803 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5804 }
5805
5806 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5807 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5808 data |= LC_DYN_LANES_PWR_STATE(3);
5809 if (data != orig)
5810 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5811
5812 if (rdev->family >= CHIP_BARTS) {
5813 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5814 data &= ~LS2_EXIT_TIME_MASK;
5815 data |= LS2_EXIT_TIME(1);
5816 if (data != orig)
5817 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5818
5819 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5820 data &= ~LS2_EXIT_TIME_MASK;
5821 data |= LS2_EXIT_TIME(1);
5822 if (data != orig)
5823 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5824 }
5825 }
5826 }
5827
5828 /* evergreen parts only */
5829 if (rdev->family < CHIP_BARTS)
5830 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5831
5832 if (pcie_lc_cntl != pcie_lc_cntl_old)
5833 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5834}
Linux 6.x block layer and io_uring tree(s)