projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/dpm: save some display parameters for DPM
[linux-2.6-block.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
5a0e3ad6 26#include <linux/slab.h>
760285e7 27#include <drm/drmP.h>
bcc1c2a1 28#include "radeon.h"
e6990375 29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
4a15903d
AD		 38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
2948f5e6
AD		 48#include "clearstate_evergreen.h"
49
50static u32 sumo_rlc_save_restore_register_list[] =
51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
134static u32 sumo_rlc_save_restore_register_list_size = ARRAY_SIZE(sumo_rlc_save_restore_register_list);
135
bcc1c2a1
AD		 136static void evergreen_gpu_init(struct radeon_device *rdev);
137void evergreen_fini(struct radeon_device *rdev);
b07759bf 138void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 139void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 140extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
141 int ring, u32 cp_int_cntl);
bcc1c2a1 142
d4788db3
AD		 143static const u32 evergreen_golden_registers[] =
144{
145 0x3f90, 0xffff0000, 0xff000000,
146 0x9148, 0xffff0000, 0xff000000,
147 0x3f94, 0xffff0000, 0xff000000,
148 0x914c, 0xffff0000, 0xff000000,
149 0x9b7c, 0xffffffff, 0x00000000,
150 0x8a14, 0xffffffff, 0x00000007,
151 0x8b10, 0xffffffff, 0x00000000,
152 0x960c, 0xffffffff, 0x54763210,
153 0x88c4, 0xffffffff, 0x000000c2,
154 0x88d4, 0xffffffff, 0x00000010,
155 0x8974, 0xffffffff, 0x00000000,
156 0xc78, 0x00000080, 0x00000080,
157 0x5eb4, 0xffffffff, 0x00000002,
158 0x5e78, 0xffffffff, 0x001000f0,
159 0x6104, 0x01000300, 0x00000000,
160 0x5bc0, 0x00300000, 0x00000000,
161 0x7030, 0xffffffff, 0x00000011,
162 0x7c30, 0xffffffff, 0x00000011,
163 0x10830, 0xffffffff, 0x00000011,
164 0x11430, 0xffffffff, 0x00000011,
165 0x12030, 0xffffffff, 0x00000011,
166 0x12c30, 0xffffffff, 0x00000011,
167 0xd02c, 0xffffffff, 0x08421000,
168 0x240c, 0xffffffff, 0x00000380,
169 0x8b24, 0xffffffff, 0x00ff0fff,
170 0x28a4c, 0x06000000, 0x06000000,
171 0x10c, 0x00000001, 0x00000001,
172 0x8d00, 0xffffffff, 0x100e4848,
173 0x8d04, 0xffffffff, 0x00164745,
174 0x8c00, 0xffffffff, 0xe4000003,
175 0x8c04, 0xffffffff, 0x40600060,
176 0x8c08, 0xffffffff, 0x001c001c,
177 0x8cf0, 0xffffffff, 0x08e00620,
178 0x8c20, 0xffffffff, 0x00800080,
179 0x8c24, 0xffffffff, 0x00800080,
180 0x8c18, 0xffffffff, 0x20202078,
181 0x8c1c, 0xffffffff, 0x00001010,
182 0x28350, 0xffffffff, 0x00000000,
183 0xa008, 0xffffffff, 0x00010000,
184 0x5cc, 0xffffffff, 0x00000001,
185 0x9508, 0xffffffff, 0x00000002,
186 0x913c, 0x0000000f, 0x0000000a
187};
188
189static const u32 evergreen_golden_registers2[] =
190{
191 0x2f4c, 0xffffffff, 0x00000000,
192 0x54f4, 0xffffffff, 0x00000000,
193 0x54f0, 0xffffffff, 0x00000000,
194 0x5498, 0xffffffff, 0x00000000,
195 0x549c, 0xffffffff, 0x00000000,
196 0x5494, 0xffffffff, 0x00000000,
197 0x53cc, 0xffffffff, 0x00000000,
198 0x53c8, 0xffffffff, 0x00000000,
199 0x53c4, 0xffffffff, 0x00000000,
200 0x53c0, 0xffffffff, 0x00000000,
201 0x53bc, 0xffffffff, 0x00000000,
202 0x53b8, 0xffffffff, 0x00000000,
203 0x53b4, 0xffffffff, 0x00000000,
204 0x53b0, 0xffffffff, 0x00000000
205};
206
207static const u32 cypress_mgcg_init[] =
208{
209 0x802c, 0xffffffff, 0xc0000000,
210 0x5448, 0xffffffff, 0x00000100,
211 0x55e4, 0xffffffff, 0x00000100,
212 0x160c, 0xffffffff, 0x00000100,
213 0x5644, 0xffffffff, 0x00000100,
214 0xc164, 0xffffffff, 0x00000100,
215 0x8a18, 0xffffffff, 0x00000100,
216 0x897c, 0xffffffff, 0x06000100,
217 0x8b28, 0xffffffff, 0x00000100,
218 0x9144, 0xffffffff, 0x00000100,
219 0x9a60, 0xffffffff, 0x00000100,
220 0x9868, 0xffffffff, 0x00000100,
221 0x8d58, 0xffffffff, 0x00000100,
222 0x9510, 0xffffffff, 0x00000100,
223 0x949c, 0xffffffff, 0x00000100,
224 0x9654, 0xffffffff, 0x00000100,
225 0x9030, 0xffffffff, 0x00000100,
226 0x9034, 0xffffffff, 0x00000100,
227 0x9038, 0xffffffff, 0x00000100,
228 0x903c, 0xffffffff, 0x00000100,
229 0x9040, 0xffffffff, 0x00000100,
230 0xa200, 0xffffffff, 0x00000100,
231 0xa204, 0xffffffff, 0x00000100,
232 0xa208, 0xffffffff, 0x00000100,
233 0xa20c, 0xffffffff, 0x00000100,
234 0x971c, 0xffffffff, 0x00000100,
235 0x977c, 0xffffffff, 0x00000100,
236 0x3f80, 0xffffffff, 0x00000100,
237 0xa210, 0xffffffff, 0x00000100,
238 0xa214, 0xffffffff, 0x00000100,
239 0x4d8, 0xffffffff, 0x00000100,
240 0x9784, 0xffffffff, 0x00000100,
241 0x9698, 0xffffffff, 0x00000100,
242 0x4d4, 0xffffffff, 0x00000200,
243 0x30cc, 0xffffffff, 0x00000100,
244 0xd0c0, 0xffffffff, 0xff000100,
245 0x802c, 0xffffffff, 0x40000000,
246 0x915c, 0xffffffff, 0x00010000,
247 0x9160, 0xffffffff, 0x00030002,
248 0x9178, 0xffffffff, 0x00070000,
249 0x917c, 0xffffffff, 0x00030002,
250 0x9180, 0xffffffff, 0x00050004,
251 0x918c, 0xffffffff, 0x00010006,
252 0x9190, 0xffffffff, 0x00090008,
253 0x9194, 0xffffffff, 0x00070000,
254 0x9198, 0xffffffff, 0x00030002,
255 0x919c, 0xffffffff, 0x00050004,
256 0x91a8, 0xffffffff, 0x00010006,
257 0x91ac, 0xffffffff, 0x00090008,
258 0x91b0, 0xffffffff, 0x00070000,
259 0x91b4, 0xffffffff, 0x00030002,
260 0x91b8, 0xffffffff, 0x00050004,
261 0x91c4, 0xffffffff, 0x00010006,
262 0x91c8, 0xffffffff, 0x00090008,
263 0x91cc, 0xffffffff, 0x00070000,
264 0x91d0, 0xffffffff, 0x00030002,
265 0x91d4, 0xffffffff, 0x00050004,
266 0x91e0, 0xffffffff, 0x00010006,
267 0x91e4, 0xffffffff, 0x00090008,
268 0x91e8, 0xffffffff, 0x00000000,
269 0x91ec, 0xffffffff, 0x00070000,
270 0x91f0, 0xffffffff, 0x00030002,
271 0x91f4, 0xffffffff, 0x00050004,
272 0x9200, 0xffffffff, 0x00010006,
273 0x9204, 0xffffffff, 0x00090008,
274 0x9208, 0xffffffff, 0x00070000,
275 0x920c, 0xffffffff, 0x00030002,
276 0x9210, 0xffffffff, 0x00050004,
277 0x921c, 0xffffffff, 0x00010006,
278 0x9220, 0xffffffff, 0x00090008,
279 0x9224, 0xffffffff, 0x00070000,
280 0x9228, 0xffffffff, 0x00030002,
281 0x922c, 0xffffffff, 0x00050004,
282 0x9238, 0xffffffff, 0x00010006,
283 0x923c, 0xffffffff, 0x00090008,
284 0x9240, 0xffffffff, 0x00070000,
285 0x9244, 0xffffffff, 0x00030002,
286 0x9248, 0xffffffff, 0x00050004,
287 0x9254, 0xffffffff, 0x00010006,
288 0x9258, 0xffffffff, 0x00090008,
289 0x925c, 0xffffffff, 0x00070000,
290 0x9260, 0xffffffff, 0x00030002,
291 0x9264, 0xffffffff, 0x00050004,
292 0x9270, 0xffffffff, 0x00010006,
293 0x9274, 0xffffffff, 0x00090008,
294 0x9278, 0xffffffff, 0x00070000,
295 0x927c, 0xffffffff, 0x00030002,
296 0x9280, 0xffffffff, 0x00050004,
297 0x928c, 0xffffffff, 0x00010006,
298 0x9290, 0xffffffff, 0x00090008,
299 0x9294, 0xffffffff, 0x00000000,
300 0x929c, 0xffffffff, 0x00000001,
301 0x802c, 0xffffffff, 0x40010000,
302 0x915c, 0xffffffff, 0x00010000,
303 0x9160, 0xffffffff, 0x00030002,
304 0x9178, 0xffffffff, 0x00070000,
305 0x917c, 0xffffffff, 0x00030002,
306 0x9180, 0xffffffff, 0x00050004,
307 0x918c, 0xffffffff, 0x00010006,
308 0x9190, 0xffffffff, 0x00090008,
309 0x9194, 0xffffffff, 0x00070000,
310 0x9198, 0xffffffff, 0x00030002,
311 0x919c, 0xffffffff, 0x00050004,
312 0x91a8, 0xffffffff, 0x00010006,
313 0x91ac, 0xffffffff, 0x00090008,
314 0x91b0, 0xffffffff, 0x00070000,
315 0x91b4, 0xffffffff, 0x00030002,
316 0x91b8, 0xffffffff, 0x00050004,
317 0x91c4, 0xffffffff, 0x00010006,
318 0x91c8, 0xffffffff, 0x00090008,
319 0x91cc, 0xffffffff, 0x00070000,
320 0x91d0, 0xffffffff, 0x00030002,
321 0x91d4, 0xffffffff, 0x00050004,
322 0x91e0, 0xffffffff, 0x00010006,
323 0x91e4, 0xffffffff, 0x00090008,
324 0x91e8, 0xffffffff, 0x00000000,
325 0x91ec, 0xffffffff, 0x00070000,
326 0x91f0, 0xffffffff, 0x00030002,
327 0x91f4, 0xffffffff, 0x00050004,
328 0x9200, 0xffffffff, 0x00010006,
329 0x9204, 0xffffffff, 0x00090008,
330 0x9208, 0xffffffff, 0x00070000,
331 0x920c, 0xffffffff, 0x00030002,
332 0x9210, 0xffffffff, 0x00050004,
333 0x921c, 0xffffffff, 0x00010006,
334 0x9220, 0xffffffff, 0x00090008,
335 0x9224, 0xffffffff, 0x00070000,
336 0x9228, 0xffffffff, 0x00030002,
337 0x922c, 0xffffffff, 0x00050004,
338 0x9238, 0xffffffff, 0x00010006,
339 0x923c, 0xffffffff, 0x00090008,
340 0x9240, 0xffffffff, 0x00070000,
341 0x9244, 0xffffffff, 0x00030002,
342 0x9248, 0xffffffff, 0x00050004,
343 0x9254, 0xffffffff, 0x00010006,
344 0x9258, 0xffffffff, 0x00090008,
345 0x925c, 0xffffffff, 0x00070000,
346 0x9260, 0xffffffff, 0x00030002,
347 0x9264, 0xffffffff, 0x00050004,
348 0x9270, 0xffffffff, 0x00010006,
349 0x9274, 0xffffffff, 0x00090008,
350 0x9278, 0xffffffff, 0x00070000,
351 0x927c, 0xffffffff, 0x00030002,
352 0x9280, 0xffffffff, 0x00050004,
353 0x928c, 0xffffffff, 0x00010006,
354 0x9290, 0xffffffff, 0x00090008,
355 0x9294, 0xffffffff, 0x00000000,
356 0x929c, 0xffffffff, 0x00000001,
357 0x802c, 0xffffffff, 0xc0000000
358};
359
360static const u32 redwood_mgcg_init[] =
361{
362 0x802c, 0xffffffff, 0xc0000000,
363 0x5448, 0xffffffff, 0x00000100,
364 0x55e4, 0xffffffff, 0x00000100,
365 0x160c, 0xffffffff, 0x00000100,
366 0x5644, 0xffffffff, 0x00000100,
367 0xc164, 0xffffffff, 0x00000100,
368 0x8a18, 0xffffffff, 0x00000100,
369 0x897c, 0xffffffff, 0x06000100,
370 0x8b28, 0xffffffff, 0x00000100,
371 0x9144, 0xffffffff, 0x00000100,
372 0x9a60, 0xffffffff, 0x00000100,
373 0x9868, 0xffffffff, 0x00000100,
374 0x8d58, 0xffffffff, 0x00000100,
375 0x9510, 0xffffffff, 0x00000100,
376 0x949c, 0xffffffff, 0x00000100,
377 0x9654, 0xffffffff, 0x00000100,
378 0x9030, 0xffffffff, 0x00000100,
379 0x9034, 0xffffffff, 0x00000100,
380 0x9038, 0xffffffff, 0x00000100,
381 0x903c, 0xffffffff, 0x00000100,
382 0x9040, 0xffffffff, 0x00000100,
383 0xa200, 0xffffffff, 0x00000100,
384 0xa204, 0xffffffff, 0x00000100,
385 0xa208, 0xffffffff, 0x00000100,
386 0xa20c, 0xffffffff, 0x00000100,
387 0x971c, 0xffffffff, 0x00000100,
388 0x977c, 0xffffffff, 0x00000100,
389 0x3f80, 0xffffffff, 0x00000100,
390 0xa210, 0xffffffff, 0x00000100,
391 0xa214, 0xffffffff, 0x00000100,
392 0x4d8, 0xffffffff, 0x00000100,
393 0x9784, 0xffffffff, 0x00000100,
394 0x9698, 0xffffffff, 0x00000100,
395 0x4d4, 0xffffffff, 0x00000200,
396 0x30cc, 0xffffffff, 0x00000100,
397 0xd0c0, 0xffffffff, 0xff000100,
398 0x802c, 0xffffffff, 0x40000000,
399 0x915c, 0xffffffff, 0x00010000,
400 0x9160, 0xffffffff, 0x00030002,
401 0x9178, 0xffffffff, 0x00070000,
402 0x917c, 0xffffffff, 0x00030002,
403 0x9180, 0xffffffff, 0x00050004,
404 0x918c, 0xffffffff, 0x00010006,
405 0x9190, 0xffffffff, 0x00090008,
406 0x9194, 0xffffffff, 0x00070000,
407 0x9198, 0xffffffff, 0x00030002,
408 0x919c, 0xffffffff, 0x00050004,
409 0x91a8, 0xffffffff, 0x00010006,
410 0x91ac, 0xffffffff, 0x00090008,
411 0x91b0, 0xffffffff, 0x00070000,
412 0x91b4, 0xffffffff, 0x00030002,
413 0x91b8, 0xffffffff, 0x00050004,
414 0x91c4, 0xffffffff, 0x00010006,
415 0x91c8, 0xffffffff, 0x00090008,
416 0x91cc, 0xffffffff, 0x00070000,
417 0x91d0, 0xffffffff, 0x00030002,
418 0x91d4, 0xffffffff, 0x00050004,
419 0x91e0, 0xffffffff, 0x00010006,
420 0x91e4, 0xffffffff, 0x00090008,
421 0x91e8, 0xffffffff, 0x00000000,
422 0x91ec, 0xffffffff, 0x00070000,
423 0x91f0, 0xffffffff, 0x00030002,
424 0x91f4, 0xffffffff, 0x00050004,
425 0x9200, 0xffffffff, 0x00010006,
426 0x9204, 0xffffffff, 0x00090008,
427 0x9294, 0xffffffff, 0x00000000,
428 0x929c, 0xffffffff, 0x00000001,
429 0x802c, 0xffffffff, 0xc0000000
430};
431
432static const u32 cedar_golden_registers[] =
433{
434 0x3f90, 0xffff0000, 0xff000000,
435 0x9148, 0xffff0000, 0xff000000,
436 0x3f94, 0xffff0000, 0xff000000,
437 0x914c, 0xffff0000, 0xff000000,
438 0x9b7c, 0xffffffff, 0x00000000,
439 0x8a14, 0xffffffff, 0x00000007,
440 0x8b10, 0xffffffff, 0x00000000,
441 0x960c, 0xffffffff, 0x54763210,
442 0x88c4, 0xffffffff, 0x000000c2,
443 0x88d4, 0xffffffff, 0x00000000,
444 0x8974, 0xffffffff, 0x00000000,
445 0xc78, 0x00000080, 0x00000080,
446 0x5eb4, 0xffffffff, 0x00000002,
447 0x5e78, 0xffffffff, 0x001000f0,
448 0x6104, 0x01000300, 0x00000000,
449 0x5bc0, 0x00300000, 0x00000000,
450 0x7030, 0xffffffff, 0x00000011,
451 0x7c30, 0xffffffff, 0x00000011,
452 0x10830, 0xffffffff, 0x00000011,
453 0x11430, 0xffffffff, 0x00000011,
454 0xd02c, 0xffffffff, 0x08421000,
455 0x240c, 0xffffffff, 0x00000380,
456 0x8b24, 0xffffffff, 0x00ff0fff,
457 0x28a4c, 0x06000000, 0x06000000,
458 0x10c, 0x00000001, 0x00000001,
459 0x8d00, 0xffffffff, 0x100e4848,
460 0x8d04, 0xffffffff, 0x00164745,
461 0x8c00, 0xffffffff, 0xe4000003,
462 0x8c04, 0xffffffff, 0x40600060,
463 0x8c08, 0xffffffff, 0x001c001c,
464 0x8cf0, 0xffffffff, 0x08e00410,
465 0x8c20, 0xffffffff, 0x00800080,
466 0x8c24, 0xffffffff, 0x00800080,
467 0x8c18, 0xffffffff, 0x20202078,
468 0x8c1c, 0xffffffff, 0x00001010,
469 0x28350, 0xffffffff, 0x00000000,
470 0xa008, 0xffffffff, 0x00010000,
471 0x5cc, 0xffffffff, 0x00000001,
472 0x9508, 0xffffffff, 0x00000002
473};
474
475static const u32 cedar_mgcg_init[] =
476{
477 0x802c, 0xffffffff, 0xc0000000,
478 0x5448, 0xffffffff, 0x00000100,
479 0x55e4, 0xffffffff, 0x00000100,
480 0x160c, 0xffffffff, 0x00000100,
481 0x5644, 0xffffffff, 0x00000100,
482 0xc164, 0xffffffff, 0x00000100,
483 0x8a18, 0xffffffff, 0x00000100,
484 0x897c, 0xffffffff, 0x06000100,
485 0x8b28, 0xffffffff, 0x00000100,
486 0x9144, 0xffffffff, 0x00000100,
487 0x9a60, 0xffffffff, 0x00000100,
488 0x9868, 0xffffffff, 0x00000100,
489 0x8d58, 0xffffffff, 0x00000100,
490 0x9510, 0xffffffff, 0x00000100,
491 0x949c, 0xffffffff, 0x00000100,
492 0x9654, 0xffffffff, 0x00000100,
493 0x9030, 0xffffffff, 0x00000100,
494 0x9034, 0xffffffff, 0x00000100,
495 0x9038, 0xffffffff, 0x00000100,
496 0x903c, 0xffffffff, 0x00000100,
497 0x9040, 0xffffffff, 0x00000100,
498 0xa200, 0xffffffff, 0x00000100,
499 0xa204, 0xffffffff, 0x00000100,
500 0xa208, 0xffffffff, 0x00000100,
501 0xa20c, 0xffffffff, 0x00000100,
502 0x971c, 0xffffffff, 0x00000100,
503 0x977c, 0xffffffff, 0x00000100,
504 0x3f80, 0xffffffff, 0x00000100,
505 0xa210, 0xffffffff, 0x00000100,
506 0xa214, 0xffffffff, 0x00000100,
507 0x4d8, 0xffffffff, 0x00000100,
508 0x9784, 0xffffffff, 0x00000100,
509 0x9698, 0xffffffff, 0x00000100,
510 0x4d4, 0xffffffff, 0x00000200,
511 0x30cc, 0xffffffff, 0x00000100,
512 0xd0c0, 0xffffffff, 0xff000100,
513 0x802c, 0xffffffff, 0x40000000,
514 0x915c, 0xffffffff, 0x00010000,
515 0x9178, 0xffffffff, 0x00050000,
516 0x917c, 0xffffffff, 0x00030002,
517 0x918c, 0xffffffff, 0x00010004,
518 0x9190, 0xffffffff, 0x00070006,
519 0x9194, 0xffffffff, 0x00050000,
520 0x9198, 0xffffffff, 0x00030002,
521 0x91a8, 0xffffffff, 0x00010004,
522 0x91ac, 0xffffffff, 0x00070006,
523 0x91e8, 0xffffffff, 0x00000000,
524 0x9294, 0xffffffff, 0x00000000,
525 0x929c, 0xffffffff, 0x00000001,
526 0x802c, 0xffffffff, 0xc0000000
527};
528
529static const u32 juniper_mgcg_init[] =
530{
531 0x802c, 0xffffffff, 0xc0000000,
532 0x5448, 0xffffffff, 0x00000100,
533 0x55e4, 0xffffffff, 0x00000100,
534 0x160c, 0xffffffff, 0x00000100,
535 0x5644, 0xffffffff, 0x00000100,
536 0xc164, 0xffffffff, 0x00000100,
537 0x8a18, 0xffffffff, 0x00000100,
538 0x897c, 0xffffffff, 0x06000100,
539 0x8b28, 0xffffffff, 0x00000100,
540 0x9144, 0xffffffff, 0x00000100,
541 0x9a60, 0xffffffff, 0x00000100,
542 0x9868, 0xffffffff, 0x00000100,
543 0x8d58, 0xffffffff, 0x00000100,
544 0x9510, 0xffffffff, 0x00000100,
545 0x949c, 0xffffffff, 0x00000100,
546 0x9654, 0xffffffff, 0x00000100,
547 0x9030, 0xffffffff, 0x00000100,
548 0x9034, 0xffffffff, 0x00000100,
549 0x9038, 0xffffffff, 0x00000100,
550 0x903c, 0xffffffff, 0x00000100,
551 0x9040, 0xffffffff, 0x00000100,
552 0xa200, 0xffffffff, 0x00000100,
553 0xa204, 0xffffffff, 0x00000100,
554 0xa208, 0xffffffff, 0x00000100,
555 0xa20c, 0xffffffff, 0x00000100,
556 0x971c, 0xffffffff, 0x00000100,
557 0xd0c0, 0xffffffff, 0xff000100,
558 0x802c, 0xffffffff, 0x40000000,
559 0x915c, 0xffffffff, 0x00010000,
560 0x9160, 0xffffffff, 0x00030002,
561 0x9178, 0xffffffff, 0x00070000,
562 0x917c, 0xffffffff, 0x00030002,
563 0x9180, 0xffffffff, 0x00050004,
564 0x918c, 0xffffffff, 0x00010006,
565 0x9190, 0xffffffff, 0x00090008,
566 0x9194, 0xffffffff, 0x00070000,
567 0x9198, 0xffffffff, 0x00030002,
568 0x919c, 0xffffffff, 0x00050004,
569 0x91a8, 0xffffffff, 0x00010006,
570 0x91ac, 0xffffffff, 0x00090008,
571 0x91b0, 0xffffffff, 0x00070000,
572 0x91b4, 0xffffffff, 0x00030002,
573 0x91b8, 0xffffffff, 0x00050004,
574 0x91c4, 0xffffffff, 0x00010006,
575 0x91c8, 0xffffffff, 0x00090008,
576 0x91cc, 0xffffffff, 0x00070000,
577 0x91d0, 0xffffffff, 0x00030002,
578 0x91d4, 0xffffffff, 0x00050004,
579 0x91e0, 0xffffffff, 0x00010006,
580 0x91e4, 0xffffffff, 0x00090008,
581 0x91e8, 0xffffffff, 0x00000000,
582 0x91ec, 0xffffffff, 0x00070000,
583 0x91f0, 0xffffffff, 0x00030002,
584 0x91f4, 0xffffffff, 0x00050004,
585 0x9200, 0xffffffff, 0x00010006,
586 0x9204, 0xffffffff, 0x00090008,
587 0x9208, 0xffffffff, 0x00070000,
588 0x920c, 0xffffffff, 0x00030002,
589 0x9210, 0xffffffff, 0x00050004,
590 0x921c, 0xffffffff, 0x00010006,
591 0x9220, 0xffffffff, 0x00090008,
592 0x9224, 0xffffffff, 0x00070000,
593 0x9228, 0xffffffff, 0x00030002,
594 0x922c, 0xffffffff, 0x00050004,
595 0x9238, 0xffffffff, 0x00010006,
596 0x923c, 0xffffffff, 0x00090008,
597 0x9240, 0xffffffff, 0x00070000,
598 0x9244, 0xffffffff, 0x00030002,
599 0x9248, 0xffffffff, 0x00050004,
600 0x9254, 0xffffffff, 0x00010006,
601 0x9258, 0xffffffff, 0x00090008,
602 0x925c, 0xffffffff, 0x00070000,
603 0x9260, 0xffffffff, 0x00030002,
604 0x9264, 0xffffffff, 0x00050004,
605 0x9270, 0xffffffff, 0x00010006,
606 0x9274, 0xffffffff, 0x00090008,
607 0x9278, 0xffffffff, 0x00070000,
608 0x927c, 0xffffffff, 0x00030002,
609 0x9280, 0xffffffff, 0x00050004,
610 0x928c, 0xffffffff, 0x00010006,
611 0x9290, 0xffffffff, 0x00090008,
612 0x9294, 0xffffffff, 0x00000000,
613 0x929c, 0xffffffff, 0x00000001,
614 0x802c, 0xffffffff, 0xc0000000,
615 0x977c, 0xffffffff, 0x00000100,
616 0x3f80, 0xffffffff, 0x00000100,
617 0xa210, 0xffffffff, 0x00000100,
618 0xa214, 0xffffffff, 0x00000100,
619 0x4d8, 0xffffffff, 0x00000100,
620 0x9784, 0xffffffff, 0x00000100,
621 0x9698, 0xffffffff, 0x00000100,
622 0x4d4, 0xffffffff, 0x00000200,
623 0x30cc, 0xffffffff, 0x00000100,
624 0x802c, 0xffffffff, 0xc0000000
625};
626
627static const u32 supersumo_golden_registers[] =
628{
629 0x5eb4, 0xffffffff, 0x00000002,
630 0x5cc, 0xffffffff, 0x00000001,
631 0x7030, 0xffffffff, 0x00000011,
632 0x7c30, 0xffffffff, 0x00000011,
633 0x6104, 0x01000300, 0x00000000,
634 0x5bc0, 0x00300000, 0x00000000,
635 0x8c04, 0xffffffff, 0x40600060,
636 0x8c08, 0xffffffff, 0x001c001c,
637 0x8c20, 0xffffffff, 0x00800080,
638 0x8c24, 0xffffffff, 0x00800080,
639 0x8c18, 0xffffffff, 0x20202078,
640 0x8c1c, 0xffffffff, 0x00001010,
641 0x918c, 0xffffffff, 0x00010006,
642 0x91a8, 0xffffffff, 0x00010006,
643 0x91c4, 0xffffffff, 0x00010006,
644 0x91e0, 0xffffffff, 0x00010006,
645 0x9200, 0xffffffff, 0x00010006,
646 0x9150, 0xffffffff, 0x6e944040,
647 0x917c, 0xffffffff, 0x00030002,
648 0x9180, 0xffffffff, 0x00050004,
649 0x9198, 0xffffffff, 0x00030002,
650 0x919c, 0xffffffff, 0x00050004,
651 0x91b4, 0xffffffff, 0x00030002,
652 0x91b8, 0xffffffff, 0x00050004,
653 0x91d0, 0xffffffff, 0x00030002,
654 0x91d4, 0xffffffff, 0x00050004,
655 0x91f0, 0xffffffff, 0x00030002,
656 0x91f4, 0xffffffff, 0x00050004,
657 0x915c, 0xffffffff, 0x00010000,
658 0x9160, 0xffffffff, 0x00030002,
659 0x3f90, 0xffff0000, 0xff000000,
660 0x9178, 0xffffffff, 0x00070000,
661 0x9194, 0xffffffff, 0x00070000,
662 0x91b0, 0xffffffff, 0x00070000,
663 0x91cc, 0xffffffff, 0x00070000,
664 0x91ec, 0xffffffff, 0x00070000,
665 0x9148, 0xffff0000, 0xff000000,
666 0x9190, 0xffffffff, 0x00090008,
667 0x91ac, 0xffffffff, 0x00090008,
668 0x91c8, 0xffffffff, 0x00090008,
669 0x91e4, 0xffffffff, 0x00090008,
670 0x9204, 0xffffffff, 0x00090008,
671 0x3f94, 0xffff0000, 0xff000000,
672 0x914c, 0xffff0000, 0xff000000,
673 0x929c, 0xffffffff, 0x00000001,
674 0x8a18, 0xffffffff, 0x00000100,
675 0x8b28, 0xffffffff, 0x00000100,
676 0x9144, 0xffffffff, 0x00000100,
677 0x5644, 0xffffffff, 0x00000100,
678 0x9b7c, 0xffffffff, 0x00000000,
679 0x8030, 0xffffffff, 0x0000100a,
680 0x8a14, 0xffffffff, 0x00000007,
681 0x8b24, 0xffffffff, 0x00ff0fff,
682 0x8b10, 0xffffffff, 0x00000000,
683 0x28a4c, 0x06000000, 0x06000000,
684 0x4d8, 0xffffffff, 0x00000100,
685 0x913c, 0xffff000f, 0x0100000a,
686 0x960c, 0xffffffff, 0x54763210,
687 0x88c4, 0xffffffff, 0x000000c2,
688 0x88d4, 0xffffffff, 0x00000010,
689 0x8974, 0xffffffff, 0x00000000,
690 0xc78, 0x00000080, 0x00000080,
691 0x5e78, 0xffffffff, 0x001000f0,
692 0xd02c, 0xffffffff, 0x08421000,
693 0xa008, 0xffffffff, 0x00010000,
694 0x8d00, 0xffffffff, 0x100e4848,
695 0x8d04, 0xffffffff, 0x00164745,
696 0x8c00, 0xffffffff, 0xe4000003,
697 0x8cf0, 0x1fffffff, 0x08e00620,
698 0x28350, 0xffffffff, 0x00000000,
699 0x9508, 0xffffffff, 0x00000002
700};
701
702static const u32 sumo_golden_registers[] =
703{
704 0x900c, 0x00ffffff, 0x0017071f,
705 0x8c18, 0xffffffff, 0x10101060,
706 0x8c1c, 0xffffffff, 0x00001010,
707 0x8c30, 0x0000000f, 0x00000005,
708 0x9688, 0x0000000f, 0x00000007
709};
710
711static const u32 wrestler_golden_registers[] =
712{
713 0x5eb4, 0xffffffff, 0x00000002,
714 0x5cc, 0xffffffff, 0x00000001,
715 0x7030, 0xffffffff, 0x00000011,
716 0x7c30, 0xffffffff, 0x00000011,
717 0x6104, 0x01000300, 0x00000000,
718 0x5bc0, 0x00300000, 0x00000000,
719 0x918c, 0xffffffff, 0x00010006,
720 0x91a8, 0xffffffff, 0x00010006,
721 0x9150, 0xffffffff, 0x6e944040,
722 0x917c, 0xffffffff, 0x00030002,
723 0x9198, 0xffffffff, 0x00030002,
724 0x915c, 0xffffffff, 0x00010000,
725 0x3f90, 0xffff0000, 0xff000000,
726 0x9178, 0xffffffff, 0x00070000,
727 0x9194, 0xffffffff, 0x00070000,
728 0x9148, 0xffff0000, 0xff000000,
729 0x9190, 0xffffffff, 0x00090008,
730 0x91ac, 0xffffffff, 0x00090008,
731 0x3f94, 0xffff0000, 0xff000000,
732 0x914c, 0xffff0000, 0xff000000,
733 0x929c, 0xffffffff, 0x00000001,
734 0x8a18, 0xffffffff, 0x00000100,
735 0x8b28, 0xffffffff, 0x00000100,
736 0x9144, 0xffffffff, 0x00000100,
737 0x9b7c, 0xffffffff, 0x00000000,
738 0x8030, 0xffffffff, 0x0000100a,
739 0x8a14, 0xffffffff, 0x00000001,
740 0x8b24, 0xffffffff, 0x00ff0fff,
741 0x8b10, 0xffffffff, 0x00000000,
742 0x28a4c, 0x06000000, 0x06000000,
743 0x4d8, 0xffffffff, 0x00000100,
744 0x913c, 0xffff000f, 0x0100000a,
745 0x960c, 0xffffffff, 0x54763210,
746 0x88c4, 0xffffffff, 0x000000c2,
747 0x88d4, 0xffffffff, 0x00000010,
748 0x8974, 0xffffffff, 0x00000000,
749 0xc78, 0x00000080, 0x00000080,
750 0x5e78, 0xffffffff, 0x001000f0,
751 0xd02c, 0xffffffff, 0x08421000,
752 0xa008, 0xffffffff, 0x00010000,
753 0x8d00, 0xffffffff, 0x100e4848,
754 0x8d04, 0xffffffff, 0x00164745,
755 0x8c00, 0xffffffff, 0xe4000003,
756 0x8cf0, 0x1fffffff, 0x08e00410,
757 0x28350, 0xffffffff, 0x00000000,
758 0x9508, 0xffffffff, 0x00000002,
759 0x900c, 0xffffffff, 0x0017071f,
760 0x8c18, 0xffffffff, 0x10101060,
761 0x8c1c, 0xffffffff, 0x00001010
762};
763
764static const u32 barts_golden_registers[] =
765{
766 0x5eb4, 0xffffffff, 0x00000002,
767 0x5e78, 0x8f311ff1, 0x001000f0,
768 0x3f90, 0xffff0000, 0xff000000,
769 0x9148, 0xffff0000, 0xff000000,
770 0x3f94, 0xffff0000, 0xff000000,
771 0x914c, 0xffff0000, 0xff000000,
772 0xc78, 0x00000080, 0x00000080,
773 0xbd4, 0x70073777, 0x00010001,
774 0xd02c, 0xbfffff1f, 0x08421000,
775 0xd0b8, 0x03773777, 0x02011003,
776 0x5bc0, 0x00200000, 0x50100000,
777 0x98f8, 0x33773777, 0x02011003,
778 0x98fc, 0xffffffff, 0x76543210,
779 0x7030, 0x31000311, 0x00000011,
780 0x2f48, 0x00000007, 0x02011003,
781 0x6b28, 0x00000010, 0x00000012,
782 0x7728, 0x00000010, 0x00000012,
783 0x10328, 0x00000010, 0x00000012,
784 0x10f28, 0x00000010, 0x00000012,
785 0x11b28, 0x00000010, 0x00000012,
786 0x12728, 0x00000010, 0x00000012,
787 0x240c, 0x000007ff, 0x00000380,
788 0x8a14, 0xf000001f, 0x00000007,
789 0x8b24, 0x3fff3fff, 0x00ff0fff,
790 0x8b10, 0x0000ff0f, 0x00000000,
791 0x28a4c, 0x07ffffff, 0x06000000,
792 0x10c, 0x00000001, 0x00010003,
793 0xa02c, 0xffffffff, 0x0000009b,
794 0x913c, 0x0000000f, 0x0100000a,
795 0x8d00, 0xffff7f7f, 0x100e4848,
796 0x8d04, 0x00ffffff, 0x00164745,
797 0x8c00, 0xfffc0003, 0xe4000003,
798 0x8c04, 0xf8ff00ff, 0x40600060,
799 0x8c08, 0x00ff00ff, 0x001c001c,
800 0x8cf0, 0x1fff1fff, 0x08e00620,
801 0x8c20, 0x0fff0fff, 0x00800080,
802 0x8c24, 0x0fff0fff, 0x00800080,
803 0x8c18, 0xffffffff, 0x20202078,
804 0x8c1c, 0x0000ffff, 0x00001010,
805 0x28350, 0x00000f01, 0x00000000,
806 0x9508, 0x3700001f, 0x00000002,
807 0x960c, 0xffffffff, 0x54763210,
808 0x88c4, 0x001f3ae3, 0x000000c2,
809 0x88d4, 0x0000001f, 0x00000010,
810 0x8974, 0xffffffff, 0x00000000
811};
812
813static const u32 turks_golden_registers[] =
814{
815 0x5eb4, 0xffffffff, 0x00000002,
816 0x5e78, 0x8f311ff1, 0x001000f0,
817 0x8c8, 0x00003000, 0x00001070,
818 0x8cc, 0x000fffff, 0x00040035,
819 0x3f90, 0xffff0000, 0xfff00000,
820 0x9148, 0xffff0000, 0xfff00000,
821 0x3f94, 0xffff0000, 0xfff00000,
822 0x914c, 0xffff0000, 0xfff00000,
823 0xc78, 0x00000080, 0x00000080,
824 0xbd4, 0x00073007, 0x00010002,
825 0xd02c, 0xbfffff1f, 0x08421000,
826 0xd0b8, 0x03773777, 0x02010002,
827 0x5bc0, 0x00200000, 0x50100000,
828 0x98f8, 0x33773777, 0x00010002,
829 0x98fc, 0xffffffff, 0x33221100,
830 0x7030, 0x31000311, 0x00000011,
831 0x2f48, 0x33773777, 0x00010002,
832 0x6b28, 0x00000010, 0x00000012,
833 0x7728, 0x00000010, 0x00000012,
834 0x10328, 0x00000010, 0x00000012,
835 0x10f28, 0x00000010, 0x00000012,
836 0x11b28, 0x00000010, 0x00000012,
837 0x12728, 0x00000010, 0x00000012,
838 0x240c, 0x000007ff, 0x00000380,
839 0x8a14, 0xf000001f, 0x00000007,
840 0x8b24, 0x3fff3fff, 0x00ff0fff,
841 0x8b10, 0x0000ff0f, 0x00000000,
842 0x28a4c, 0x07ffffff, 0x06000000,
843 0x10c, 0x00000001, 0x00010003,
844 0xa02c, 0xffffffff, 0x0000009b,
845 0x913c, 0x0000000f, 0x0100000a,
846 0x8d00, 0xffff7f7f, 0x100e4848,
847 0x8d04, 0x00ffffff, 0x00164745,
848 0x8c00, 0xfffc0003, 0xe4000003,
849 0x8c04, 0xf8ff00ff, 0x40600060,
850 0x8c08, 0x00ff00ff, 0x001c001c,
851 0x8cf0, 0x1fff1fff, 0x08e00410,
852 0x8c20, 0x0fff0fff, 0x00800080,
853 0x8c24, 0x0fff0fff, 0x00800080,
854 0x8c18, 0xffffffff, 0x20202078,
855 0x8c1c, 0x0000ffff, 0x00001010,
856 0x28350, 0x00000f01, 0x00000000,
857 0x9508, 0x3700001f, 0x00000002,
858 0x960c, 0xffffffff, 0x54763210,
859 0x88c4, 0x001f3ae3, 0x000000c2,
860 0x88d4, 0x0000001f, 0x00000010,
861 0x8974, 0xffffffff, 0x00000000
862};
863
864static const u32 caicos_golden_registers[] =
865{
866 0x5eb4, 0xffffffff, 0x00000002,
867 0x5e78, 0x8f311ff1, 0x001000f0,
868 0x8c8, 0x00003420, 0x00001450,
869 0x8cc, 0x000fffff, 0x00040035,
870 0x3f90, 0xffff0000, 0xfffc0000,
871 0x9148, 0xffff0000, 0xfffc0000,
872 0x3f94, 0xffff0000, 0xfffc0000,
873 0x914c, 0xffff0000, 0xfffc0000,
874 0xc78, 0x00000080, 0x00000080,
875 0xbd4, 0x00073007, 0x00010001,
876 0xd02c, 0xbfffff1f, 0x08421000,
877 0xd0b8, 0x03773777, 0x02010001,
878 0x5bc0, 0x00200000, 0x50100000,
879 0x98f8, 0x33773777, 0x02010001,
880 0x98fc, 0xffffffff, 0x33221100,
881 0x7030, 0x31000311, 0x00000011,
882 0x2f48, 0x33773777, 0x02010001,
883 0x6b28, 0x00000010, 0x00000012,
884 0x7728, 0x00000010, 0x00000012,
885 0x10328, 0x00000010, 0x00000012,
886 0x10f28, 0x00000010, 0x00000012,
887 0x11b28, 0x00000010, 0x00000012,
888 0x12728, 0x00000010, 0x00000012,
889 0x240c, 0x000007ff, 0x00000380,
890 0x8a14, 0xf000001f, 0x00000001,
891 0x8b24, 0x3fff3fff, 0x00ff0fff,
892 0x8b10, 0x0000ff0f, 0x00000000,
893 0x28a4c, 0x07ffffff, 0x06000000,
894 0x10c, 0x00000001, 0x00010003,
895 0xa02c, 0xffffffff, 0x0000009b,
896 0x913c, 0x0000000f, 0x0100000a,
897 0x8d00, 0xffff7f7f, 0x100e4848,
898 0x8d04, 0x00ffffff, 0x00164745,
899 0x8c00, 0xfffc0003, 0xe4000003,
900 0x8c04, 0xf8ff00ff, 0x40600060,
901 0x8c08, 0x00ff00ff, 0x001c001c,
902 0x8cf0, 0x1fff1fff, 0x08e00410,
903 0x8c20, 0x0fff0fff, 0x00800080,
904 0x8c24, 0x0fff0fff, 0x00800080,
905 0x8c18, 0xffffffff, 0x20202078,
906 0x8c1c, 0x0000ffff, 0x00001010,
907 0x28350, 0x00000f01, 0x00000000,
908 0x9508, 0x3700001f, 0x00000002,
909 0x960c, 0xffffffff, 0x54763210,
910 0x88c4, 0x001f3ae3, 0x000000c2,
911 0x88d4, 0x0000001f, 0x00000010,
912 0x8974, 0xffffffff, 0x00000000
913};
914
915static void evergreen_init_golden_registers(struct radeon_device *rdev)
916{
917 switch (rdev->family) {
918 case CHIP_CYPRESS:
919 case CHIP_HEMLOCK:
920 radeon_program_register_sequence(rdev,
921 evergreen_golden_registers,
922 (const u32)ARRAY_SIZE(evergreen_golden_registers));
923 radeon_program_register_sequence(rdev,
924 evergreen_golden_registers2,
925 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
926 radeon_program_register_sequence(rdev,
927 cypress_mgcg_init,
928 (const u32)ARRAY_SIZE(cypress_mgcg_init));
929 break;
930 case CHIP_JUNIPER:
931 radeon_program_register_sequence(rdev,
932 evergreen_golden_registers,
933 (const u32)ARRAY_SIZE(evergreen_golden_registers));
934 radeon_program_register_sequence(rdev,
935 evergreen_golden_registers2,
936 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
937 radeon_program_register_sequence(rdev,
938 juniper_mgcg_init,
939 (const u32)ARRAY_SIZE(juniper_mgcg_init));
940 break;
941 case CHIP_REDWOOD:
942 radeon_program_register_sequence(rdev,
943 evergreen_golden_registers,
944 (const u32)ARRAY_SIZE(evergreen_golden_registers));
945 radeon_program_register_sequence(rdev,
946 evergreen_golden_registers2,
947 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
948 radeon_program_register_sequence(rdev,
949 redwood_mgcg_init,
950 (const u32)ARRAY_SIZE(redwood_mgcg_init));
951 break;
952 case CHIP_CEDAR:
953 radeon_program_register_sequence(rdev,
954 cedar_golden_registers,
955 (const u32)ARRAY_SIZE(cedar_golden_registers));
956 radeon_program_register_sequence(rdev,
957 evergreen_golden_registers2,
958 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
959 radeon_program_register_sequence(rdev,
960 cedar_mgcg_init,
961 (const u32)ARRAY_SIZE(cedar_mgcg_init));
962 break;
963 case CHIP_PALM:
964 radeon_program_register_sequence(rdev,
965 wrestler_golden_registers,
966 (const u32)ARRAY_SIZE(wrestler_golden_registers));
967 break;
968 case CHIP_SUMO:
969 radeon_program_register_sequence(rdev,
970 supersumo_golden_registers,
971 (const u32)ARRAY_SIZE(supersumo_golden_registers));
972 break;
973 case CHIP_SUMO2:
974 radeon_program_register_sequence(rdev,
975 supersumo_golden_registers,
976 (const u32)ARRAY_SIZE(supersumo_golden_registers));
977 radeon_program_register_sequence(rdev,
978 sumo_golden_registers,
979 (const u32)ARRAY_SIZE(sumo_golden_registers));
980 break;
981 case CHIP_BARTS:
982 radeon_program_register_sequence(rdev,
983 barts_golden_registers,
984 (const u32)ARRAY_SIZE(barts_golden_registers));
985 break;
986 case CHIP_TURKS:
987 radeon_program_register_sequence(rdev,
988 turks_golden_registers,
989 (const u32)ARRAY_SIZE(turks_golden_registers));
990 break;
991 case CHIP_CAICOS:
992 radeon_program_register_sequence(rdev,
993 caicos_golden_registers,
994 (const u32)ARRAY_SIZE(caicos_golden_registers));
995 break;
996 default:
997 break;
998 }
999}
1000
285484e2
JG		 1001void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1002 unsigned *bankh, unsigned *mtaspect,
1003 unsigned *tile_split)
1004{
1005 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1006 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1007 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1008 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1009 switch (*bankw) {
1010 default:
1011 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1012 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1013 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1014 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1015 }
1016 switch (*bankh) {
1017 default:
1018 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1019 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1020 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1021 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1022 }
1023 switch (*mtaspect) {
1024 default:
1025 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1026 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1027 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1028 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1029 }
1030}
1031
23d33ba3
AD		 1032static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1033 u32 cntl_reg, u32 status_reg)
1034{
1035 int r, i;
1036 struct atom_clock_dividers dividers;
1037
1038 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1039 clock, false, &dividers);
1040 if (r)
1041 return r;
1042
1043 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1044
1045 for (i = 0; i < 100; i++) {
1046 if (RREG32(status_reg) & DCLK_STATUS)
1047 break;
1048 mdelay(10);
1049 }
1050 if (i == 100)
1051 return -ETIMEDOUT;
1052
1053 return 0;
1054}
1055
1056int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1057{
1058 int r = 0;
1059 u32 cg_scratch = RREG32(CG_SCRATCH1);
1060
1061 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1062 if (r)
1063 goto done;
1064 cg_scratch &= 0xffff0000;
1065 cg_scratch |= vclk / 100; /* Mhz */
1066
1067 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1068 if (r)
1069 goto done;
1070 cg_scratch &= 0x0000ffff;
1071 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1072
1073done:
1074 WREG32(CG_SCRATCH1, cg_scratch);
1075
1076 return r;
1077}
1078
a8b4925c
AD		 1079int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1080{
1081 /* start off with something large */
facd112d 1082 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1083 int r;
1084
4ed10835
CK		 1085 /* bypass vclk and dclk with bclk */
1086 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1087 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1088 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1089
1090 /* put PLL in bypass mode */
1091 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1092
1093 if (!vclk || !dclk) {
1094 /* keep the Bypass mode, put PLL to sleep */
1095 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1096 return 0;
1097 }
1098
facd112d
CK		 1099 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1100 16384, 0x03FFFFFF, 0, 128, 5,
1101 &fb_div, &vclk_div, &dclk_div);
1102 if (r)
1103 return r;
a8b4925c
AD		 1104
1105 /* set VCO_MODE to 1 */
1106 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1107
1108 /* toggle UPLL_SLEEP to 1 then back to 0 */
1109 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1110 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1111
1112 /* deassert UPLL_RESET */
1113 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1114
1115 mdelay(1);
1116
facd112d 1117 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1118 if (r)
1119 return r;
1120
1121 /* assert UPLL_RESET again */
1122 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1123
1124 /* disable spread spectrum. */
1125 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1126
1127 /* set feedback divider */
facd112d 1128 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1129
1130 /* set ref divider to 0 */
1131 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1132
facd112d 1133 if (fb_div < 307200)
a8b4925c
AD		 1134 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1135 else
1136 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1137
1138 /* set PDIV_A and PDIV_B */
1139 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1140 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1141 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1142
1143 /* give the PLL some time to settle */
1144 mdelay(15);
1145
1146 /* deassert PLL_RESET */
1147 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1148
1149 mdelay(15);
1150
1151 /* switch from bypass mode to normal mode */
1152 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1153
facd112d 1154 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1155 if (r)
1156 return r;
1157
1158 /* switch VCLK and DCLK selection */
1159 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1160 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1161 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1162
1163 mdelay(100);
1164
1165 return 0;
1166}
1167
d054ac16
AD		 1168void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1169{
1170 u16 ctl, v;
32195aec 1171 int err;
d054ac16 1172
32195aec 1173 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
d054ac16
AD		 1174 if (err)
1175 return;
1176
1177 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1178
1179 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1180 * to avoid hangs or perfomance issues
1181 */
1182 if ((v == 0) || (v == 6) || (v == 7)) {
1183 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1184 ctl |= (2 << 12);
32195aec 1185 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
d054ac16
AD		 1186 }
1187}
1188
10257a6d
AD		 1189static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1190{
1191 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1192 return true;
1193 else
1194 return false;
1195}
1196
1197static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1198{
1199 u32 pos1, pos2;
1200
1201 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1202 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1203
1204 if (pos1 != pos2)
1205 return true;
1206 else
1207 return false;
1208}
1209
377edc8b
AD		 1210/**
1211 * dce4_wait_for_vblank - vblank wait asic callback.
1212 *
1213 * @rdev: radeon_device pointer
1214 * @crtc: crtc to wait for vblank on
1215 *
1216 * Wait for vblank on the requested crtc (evergreen+).
1217 */
3ae19b75
AD		 1218void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1219{
10257a6d 1220 unsigned i = 0;
3ae19b75 1221
4a15903d
AD		 1222 if (crtc >= rdev->num_crtc)
1223 return;
1224
10257a6d
AD		 1225 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1226 return;
1227
1228 /* depending on when we hit vblank, we may be close to active; if so,
1229 * wait for another frame.
1230 */
1231 while (dce4_is_in_vblank(rdev, crtc)) {
1232 if (i++ % 100 == 0) {
1233 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1234 break;
3ae19b75 1235 }
10257a6d
AD		 1236 }
1237
1238 while (!dce4_is_in_vblank(rdev, crtc)) {
1239 if (i++ % 100 == 0) {
1240 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1241 break;
3ae19b75
AD		 1242 }
1243 }
1244}
1245
377edc8b
AD		 1246/**
1247 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1248 *
1249 * @rdev: radeon_device pointer
1250 * @crtc: crtc to prepare for pageflip on
1251 *
1252 * Pre-pageflip callback (evergreen+).
1253 * Enables the pageflip irq (vblank irq).
1254 */
6f34be50
AD		 1255void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1256{
6f34be50
AD		 1257 /* enable the pflip int */
1258 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1259}
1260
377edc8b
AD		 1261/**
1262 * evergreen_post_page_flip - pos-pageflip callback.
1263 *
1264 * @rdev: radeon_device pointer
1265 * @crtc: crtc to cleanup pageflip on
1266 *
1267 * Post-pageflip callback (evergreen+).
1268 * Disables the pageflip irq (vblank irq).
1269 */
6f34be50
AD		 1270void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1271{
1272 /* disable the pflip int */
1273 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1274}
1275
377edc8b
AD		 1276/**
1277 * evergreen_page_flip - pageflip callback.
1278 *
1279 * @rdev: radeon_device pointer
1280 * @crtc_id: crtc to cleanup pageflip on
1281 * @crtc_base: new address of the crtc (GPU MC address)
1282 *
1283 * Does the actual pageflip (evergreen+).
1284 * During vblank we take the crtc lock and wait for the update_pending
1285 * bit to go high, when it does, we release the lock, and allow the
1286 * double buffered update to take place.
1287 * Returns the current update pending status.
1288 */
6f34be50
AD		 1289u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1290{
1291 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1292 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1293 int i;
6f34be50
AD		 1294
1295 /* Lock the graphics update lock */
1296 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1297 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1298
1299 /* update the scanout addresses */
1300 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1301 upper_32_bits(crtc_base));
1302 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1303 (u32)crtc_base);
1304
1305 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1306 upper_32_bits(crtc_base));
1307 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1308 (u32)crtc_base);
1309
1310 /* Wait for update_pending to go high. */
f6496479
AD		 1311 for (i = 0; i < rdev->usec_timeout; i++) {
1312 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1313 break;
1314 udelay(1);
1315 }
6f34be50
AD		 1316 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1317
1318 /* Unlock the lock, so double-buffering can take place inside vblank */
1319 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1320 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1321
1322 /* Return current update_pending status: */
1323 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1324}
1325
21a8122a 1326/* get temperature in millidegrees */
20d391d7 1327int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1328{
1c88d74f
AD		 1329 u32 temp, toffset;
1330 int actual_temp = 0;
67b3f823
AD		 1331
1332 if (rdev->family == CHIP_JUNIPER) {
1333 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1334 TOFFSET_SHIFT;
1335 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1336 TS0_ADC_DOUT_SHIFT;
1337
1338 if (toffset & 0x100)
1339 actual_temp = temp / 2 - (0x200 - toffset);
1340 else
1341 actual_temp = temp / 2 + toffset;
1342
1343 actual_temp = actual_temp * 1000;
1344
1345 } else {
1346 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1347 ASIC_T_SHIFT;
1348
1349 if (temp & 0x400)
1350 actual_temp = -256;
1351 else if (temp & 0x200)
1352 actual_temp = 255;
1353 else if (temp & 0x100) {
1354 actual_temp = temp & 0x1ff;
1355 actual_temp |= ~0x1ff;
1356 } else
1357 actual_temp = temp & 0xff;
1358
1359 actual_temp = (actual_temp * 1000) / 2;
1360 }
21a8122a 1361
67b3f823 1362 return actual_temp;
21a8122a
AD		 1363}
1364
20d391d7 1365int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1366{
1367 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1368 int actual_temp = temp - 49;
e33df25f
AD		 1369
1370 return actual_temp * 1000;
1371}
1372
377edc8b
AD		 1373/**
1374 * sumo_pm_init_profile - Initialize power profiles callback.
1375 *
1376 * @rdev: radeon_device pointer
1377 *
1378 * Initialize the power states used in profile mode
1379 * (sumo, trinity, SI).
1380 * Used for profile mode only.
1381 */
a4c9e2ee
AD		 1382void sumo_pm_init_profile(struct radeon_device *rdev)
1383{
1384 int idx;
1385
1386 /* default */
1387 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1388 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1389 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1390 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1391
1392 /* low,mid sh/mh */
1393 if (rdev->flags & RADEON_IS_MOBILITY)
1394 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1395 else
1396 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1397
1398 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1399 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1400 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1401 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1402
1403 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1404 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1405 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1406 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1407
1408 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1409 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1410 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1411 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1412
1413 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1414 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1416 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1417
1418 /* high sh/mh */
1419 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1420 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1421 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1422 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1423 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1424 rdev->pm.power_state[idx].num_clock_modes - 1;
1425
1426 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1427 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1428 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1429 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1430 rdev->pm.power_state[idx].num_clock_modes - 1;
1431}
1432
27810fb2
AD		 1433/**
1434 * btc_pm_init_profile - Initialize power profiles callback.
1435 *
1436 * @rdev: radeon_device pointer
1437 *
1438 * Initialize the power states used in profile mode
1439 * (BTC, cayman).
1440 * Used for profile mode only.
1441 */
1442void btc_pm_init_profile(struct radeon_device *rdev)
1443{
1444 int idx;
1445
1446 /* default */
1447 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1448 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1450 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1451 /* starting with BTC, there is one state that is used for both
1452 * MH and SH. Difference is that we always use the high clock index for
1453 * mclk.
1454 */
1455 if (rdev->flags & RADEON_IS_MOBILITY)
1456 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1457 else
1458 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1459 /* low sh */
1460 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1462 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1463 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1464 /* mid sh */
1465 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1466 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1468 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1469 /* high sh */
1470 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1471 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1472 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1473 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1474 /* low mh */
1475 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1476 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1477 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1478 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1479 /* mid mh */
1480 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1481 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1483 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1484 /* high mh */
1485 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1486 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1488 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1489}
1490
377edc8b
AD		 1491/**
1492 * evergreen_pm_misc - set additional pm hw parameters callback.
1493 *
1494 * @rdev: radeon_device pointer
1495 *
1496 * Set non-clock parameters associated with a power state
1497 * (voltage, etc.) (evergreen+).
1498 */
49e02b73
AD		 1499void evergreen_pm_misc(struct radeon_device *rdev)
1500{
a081a9d6
RM		 1501 int req_ps_idx = rdev->pm.requested_power_state_index;
1502 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1503 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1504 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1505
2feea49a 1506 if (voltage->type == VOLTAGE_SW) {
a377e187
AD		 1507 /* 0xff01 is a flag rather then an actual voltage */
1508 if (voltage->voltage == 0xff01)
1509 return;
2feea49a 1510 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1511 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1512 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1513 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1514 }
7ae764b1
AD		 1515
1516 /* starting with BTC, there is one state that is used for both
1517 * MH and SH. Difference is that we always use the high clock index for
1518 * mclk and vddci.
1519 */
1520 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1521 (rdev->family >= CHIP_BARTS) &&
1522 rdev->pm.active_crtc_count &&
1523 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1524 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1525 voltage = &rdev->pm.power_state[req_ps_idx].
1526 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1527
a377e187
AD		 1528 /* 0xff01 is a flag rather then an actual voltage */
1529 if (voltage->vddci == 0xff01)
1530 return;
2feea49a
AD		 1531 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1532 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1533 rdev->pm.current_vddci = voltage->vddci;
1534 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1535 }
1536 }
49e02b73
AD		 1537}
1538
377edc8b
AD		 1539/**
1540 * evergreen_pm_prepare - pre-power state change callback.
1541 *
1542 * @rdev: radeon_device pointer
1543 *
1544 * Prepare for a power state change (evergreen+).
1545 */
49e02b73
AD		 1546void evergreen_pm_prepare(struct radeon_device *rdev)
1547{
1548 struct drm_device *ddev = rdev->ddev;
1549 struct drm_crtc *crtc;
1550 struct radeon_crtc *radeon_crtc;
1551 u32 tmp;
1552
1553 /* disable any active CRTCs */
1554 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1555 radeon_crtc = to_radeon_crtc(crtc);
1556 if (radeon_crtc->enabled) {
1557 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1558 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1559 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1560 }
1561 }
1562}
1563
377edc8b
AD		 1564/**
1565 * evergreen_pm_finish - post-power state change callback.
1566 *
1567 * @rdev: radeon_device pointer
1568 *
1569 * Clean up after a power state change (evergreen+).
1570 */
49e02b73
AD		 1571void evergreen_pm_finish(struct radeon_device *rdev)
1572{
1573 struct drm_device *ddev = rdev->ddev;
1574 struct drm_crtc *crtc;
1575 struct radeon_crtc *radeon_crtc;
1576 u32 tmp;
1577
1578 /* enable any active CRTCs */
1579 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1580 radeon_crtc = to_radeon_crtc(crtc);
1581 if (radeon_crtc->enabled) {
1582 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1583 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1584 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1585 }
1586 }
1587}
1588
377edc8b
AD		 1589/**
1590 * evergreen_hpd_sense - hpd sense callback.
1591 *
1592 * @rdev: radeon_device pointer
1593 * @hpd: hpd (hotplug detect) pin
1594 *
1595 * Checks if a digital monitor is connected (evergreen+).
1596 * Returns true if connected, false if not connected.
1597 */
bcc1c2a1
AD		 1598bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1599{
1600 bool connected = false;
0ca2ab52
AD		 1601
1602 switch (hpd) {
1603 case RADEON_HPD_1:
1604 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1605 connected = true;
1606 break;
1607 case RADEON_HPD_2:
1608 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1609 connected = true;
1610 break;
1611 case RADEON_HPD_3:
1612 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1613 connected = true;
1614 break;
1615 case RADEON_HPD_4:
1616 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1617 connected = true;
1618 break;
1619 case RADEON_HPD_5:
1620 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1621 connected = true;
1622 break;
1623 case RADEON_HPD_6:
1624 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1625 connected = true;
1626 break;
1627 default:
1628 break;
1629 }
1630
bcc1c2a1
AD		 1631 return connected;
1632}
1633
377edc8b
AD		 1634/**
1635 * evergreen_hpd_set_polarity - hpd set polarity callback.
1636 *
1637 * @rdev: radeon_device pointer
1638 * @hpd: hpd (hotplug detect) pin
1639 *
1640 * Set the polarity of the hpd pin (evergreen+).
1641 */
bcc1c2a1
AD		 1642void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1643 enum radeon_hpd_id hpd)
1644{
0ca2ab52
AD		 1645 u32 tmp;
1646 bool connected = evergreen_hpd_sense(rdev, hpd);
1647
1648 switch (hpd) {
1649 case RADEON_HPD_1:
1650 tmp = RREG32(DC_HPD1_INT_CONTROL);
1651 if (connected)
1652 tmp &= ~DC_HPDx_INT_POLARITY;
1653 else
1654 tmp |= DC_HPDx_INT_POLARITY;
1655 WREG32(DC_HPD1_INT_CONTROL, tmp);
1656 break;
1657 case RADEON_HPD_2:
1658 tmp = RREG32(DC_HPD2_INT_CONTROL);
1659 if (connected)
1660 tmp &= ~DC_HPDx_INT_POLARITY;
1661 else
1662 tmp |= DC_HPDx_INT_POLARITY;
1663 WREG32(DC_HPD2_INT_CONTROL, tmp);
1664 break;
1665 case RADEON_HPD_3:
1666 tmp = RREG32(DC_HPD3_INT_CONTROL);
1667 if (connected)
1668 tmp &= ~DC_HPDx_INT_POLARITY;
1669 else
1670 tmp |= DC_HPDx_INT_POLARITY;
1671 WREG32(DC_HPD3_INT_CONTROL, tmp);
1672 break;
1673 case RADEON_HPD_4:
1674 tmp = RREG32(DC_HPD4_INT_CONTROL);
1675 if (connected)
1676 tmp &= ~DC_HPDx_INT_POLARITY;
1677 else
1678 tmp |= DC_HPDx_INT_POLARITY;
1679 WREG32(DC_HPD4_INT_CONTROL, tmp);
1680 break;
1681 case RADEON_HPD_5:
1682 tmp = RREG32(DC_HPD5_INT_CONTROL);
1683 if (connected)
1684 tmp &= ~DC_HPDx_INT_POLARITY;
1685 else
1686 tmp |= DC_HPDx_INT_POLARITY;
1687 WREG32(DC_HPD5_INT_CONTROL, tmp);
1688 break;
1689 case RADEON_HPD_6:
1690 tmp = RREG32(DC_HPD6_INT_CONTROL);
1691 if (connected)
1692 tmp &= ~DC_HPDx_INT_POLARITY;
1693 else
1694 tmp |= DC_HPDx_INT_POLARITY;
1695 WREG32(DC_HPD6_INT_CONTROL, tmp);
1696 break;
1697 default:
1698 break;
1699 }
bcc1c2a1
AD		 1700}
1701
377edc8b
AD		 1702/**
1703 * evergreen_hpd_init - hpd setup callback.
1704 *
1705 * @rdev: radeon_device pointer
1706 *
1707 * Setup the hpd pins used by the card (evergreen+).
1708 * Enable the pin, set the polarity, and enable the hpd interrupts.
1709 */
bcc1c2a1
AD		 1710void evergreen_hpd_init(struct radeon_device *rdev)
1711{
0ca2ab52
AD		 1712 struct drm_device *dev = rdev->ddev;
1713 struct drm_connector *connector;
fb98257a 1714 unsigned enabled = 0;
0ca2ab52
AD		 1715 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1716 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1717
0ca2ab52
AD		 1718 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1719 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1720
1721 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1722 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1723 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1724 * aux dp channel on imac and help (but not completely fix)
1725 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1726 * also avoid interrupt storms during dpms.
1727 */
1728 continue;
1729 }
0ca2ab52
AD		 1730 switch (radeon_connector->hpd.hpd) {
1731 case RADEON_HPD_1:
1732 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1733 break;
1734 case RADEON_HPD_2:
1735 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1736 break;
1737 case RADEON_HPD_3:
1738 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1739 break;
1740 case RADEON_HPD_4:
1741 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1742 break;
1743 case RADEON_HPD_5:
1744 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1745 break;
1746 case RADEON_HPD_6:
1747 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1748 break;
1749 default:
1750 break;
1751 }
64912e99 1752 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1753 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1754 }
fb98257a 1755 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1756}
1757
377edc8b
AD		 1758/**
1759 * evergreen_hpd_fini - hpd tear down callback.
1760 *
1761 * @rdev: radeon_device pointer
1762 *
1763 * Tear down the hpd pins used by the card (evergreen+).
1764 * Disable the hpd interrupts.
1765 */
0ca2ab52 1766void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1767{
0ca2ab52
AD		 1768 struct drm_device *dev = rdev->ddev;
1769 struct drm_connector *connector;
fb98257a 1770 unsigned disabled = 0;
0ca2ab52
AD		 1771
1772 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1773 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1774 switch (radeon_connector->hpd.hpd) {
1775 case RADEON_HPD_1:
1776 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1777 break;
1778 case RADEON_HPD_2:
1779 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1780 break;
1781 case RADEON_HPD_3:
1782 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1783 break;
1784 case RADEON_HPD_4:
1785 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1786 break;
1787 case RADEON_HPD_5:
1788 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1789 break;
1790 case RADEON_HPD_6:
1791 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1792 break;
1793 default:
1794 break;
1795 }
fb98257a 1796 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1797 }
fb98257a 1798 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1799}
1800
f9d9c362
AD		 1801/* watermark setup */
1802
1803static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1804 struct radeon_crtc *radeon_crtc,
1805 struct drm_display_mode *mode,
1806 struct drm_display_mode *other_mode)
1807{
12dfc843 1808 u32 tmp;
f9d9c362
AD		 1809 /*
1810 * Line Buffer Setup
1811 * There are 3 line buffers, each one shared by 2 display controllers.
1812 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1813 * the display controllers. The paritioning is done via one of four
1814 * preset allocations specified in bits 2:0:
1815 * first display controller
1816 * 0 - first half of lb (3840 * 2)
1817 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1818 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1819 * 3 - first 1/4 of lb (1920 * 2)
1820 * second display controller
1821 * 4 - second half of lb (3840 * 2)
1822 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1823 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1824 * 7 - last 1/4 of lb (1920 * 2)
1825 */
12dfc843
AD		 1826 /* this can get tricky if we have two large displays on a paired group
1827 * of crtcs. Ideally for multiple large displays we'd assign them to
1828 * non-linked crtcs for maximum line buffer allocation.
1829 */
1830 if (radeon_crtc->base.enabled && mode) {
1831 if (other_mode)
f9d9c362 1832 tmp = 0; /* 1/2 */
12dfc843
AD		 1833 else
1834 tmp = 2; /* whole */
1835 } else
1836 tmp = 0;
f9d9c362
AD		 1837
1838 /* second controller of the pair uses second half of the lb */
1839 if (radeon_crtc->crtc_id % 2)
1840 tmp += 4;
1841 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1842
12dfc843
AD		 1843 if (radeon_crtc->base.enabled && mode) {
1844 switch (tmp) {
1845 case 0:
1846 case 4:
1847 default:
1848 if (ASIC_IS_DCE5(rdev))
1849 return 4096 * 2;
1850 else
1851 return 3840 * 2;
1852 case 1:
1853 case 5:
1854 if (ASIC_IS_DCE5(rdev))
1855 return 6144 * 2;
1856 else
1857 return 5760 * 2;
1858 case 2:
1859 case 6:
1860 if (ASIC_IS_DCE5(rdev))
1861 return 8192 * 2;
1862 else
1863 return 7680 * 2;
1864 case 3:
1865 case 7:
1866 if (ASIC_IS_DCE5(rdev))
1867 return 2048 * 2;
1868 else
1869 return 1920 * 2;
1870 }
f9d9c362 1871 }
12dfc843
AD		 1872
1873 /* controller not enabled, so no lb used */
1874 return 0;
f9d9c362
AD		 1875}
1876
ca7db22b 1877u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1878{
1879 u32 tmp = RREG32(MC_SHARED_CHMAP);
1880
1881 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1882 case 0:
1883 default:
1884 return 1;
1885 case 1:
1886 return 2;
1887 case 2:
1888 return 4;
1889 case 3:
1890 return 8;
1891 }
1892}
1893
1894struct evergreen_wm_params {
1895 u32 dram_channels; /* number of dram channels */
1896 u32 yclk; /* bandwidth per dram data pin in kHz */
1897 u32 sclk; /* engine clock in kHz */
1898 u32 disp_clk; /* display clock in kHz */
1899 u32 src_width; /* viewport width */
1900 u32 active_time; /* active display time in ns */
1901 u32 blank_time; /* blank time in ns */
1902 bool interlaced; /* mode is interlaced */
1903 fixed20_12 vsc; /* vertical scale ratio */
1904 u32 num_heads; /* number of active crtcs */
1905 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1906 u32 lb_size; /* line buffer allocated to pipe */
1907 u32 vtaps; /* vertical scaler taps */
1908};
1909
1910static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1911{
1912 /* Calculate DRAM Bandwidth and the part allocated to display. */
1913 fixed20_12 dram_efficiency; /* 0.7 */
1914 fixed20_12 yclk, dram_channels, bandwidth;
1915 fixed20_12 a;
1916
1917 a.full = dfixed_const(1000);
1918 yclk.full = dfixed_const(wm->yclk);
1919 yclk.full = dfixed_div(yclk, a);
1920 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1921 a.full = dfixed_const(10);
1922 dram_efficiency.full = dfixed_const(7);
1923 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1924 bandwidth.full = dfixed_mul(dram_channels, yclk);
1925 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1926
1927 return dfixed_trunc(bandwidth);
1928}
1929
1930static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1931{
1932 /* Calculate DRAM Bandwidth and the part allocated to display. */
1933 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1934 fixed20_12 yclk, dram_channels, bandwidth;
1935 fixed20_12 a;
1936
1937 a.full = dfixed_const(1000);
1938 yclk.full = dfixed_const(wm->yclk);
1939 yclk.full = dfixed_div(yclk, a);
1940 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1941 a.full = dfixed_const(10);
1942 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1943 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1944 bandwidth.full = dfixed_mul(dram_channels, yclk);
1945 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1946
1947 return dfixed_trunc(bandwidth);
1948}
1949
1950static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1951{
1952 /* Calculate the display Data return Bandwidth */
1953 fixed20_12 return_efficiency; /* 0.8 */
1954 fixed20_12 sclk, bandwidth;
1955 fixed20_12 a;
1956
1957 a.full = dfixed_const(1000);
1958 sclk.full = dfixed_const(wm->sclk);
1959 sclk.full = dfixed_div(sclk, a);
1960 a.full = dfixed_const(10);
1961 return_efficiency.full = dfixed_const(8);
1962 return_efficiency.full = dfixed_div(return_efficiency, a);
1963 a.full = dfixed_const(32);
1964 bandwidth.full = dfixed_mul(a, sclk);
1965 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1966
1967 return dfixed_trunc(bandwidth);
1968}
1969
1970static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1971{
1972 /* Calculate the DMIF Request Bandwidth */
1973 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1974 fixed20_12 disp_clk, bandwidth;
1975 fixed20_12 a;
1976
1977 a.full = dfixed_const(1000);
1978 disp_clk.full = dfixed_const(wm->disp_clk);
1979 disp_clk.full = dfixed_div(disp_clk, a);
1980 a.full = dfixed_const(10);
1981 disp_clk_request_efficiency.full = dfixed_const(8);
1982 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1983 a.full = dfixed_const(32);
1984 bandwidth.full = dfixed_mul(a, disp_clk);
1985 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1986
1987 return dfixed_trunc(bandwidth);
1988}
1989
1990static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1991{
1992 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1993 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1994 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1995 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1996
1997 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1998}
1999
2000static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2001{
2002 /* Calculate the display mode Average Bandwidth
2003 * DisplayMode should contain the source and destination dimensions,
2004 * timing, etc.
2005 */
2006 fixed20_12 bpp;
2007 fixed20_12 line_time;
2008 fixed20_12 src_width;
2009 fixed20_12 bandwidth;
2010 fixed20_12 a;
2011
2012 a.full = dfixed_const(1000);
2013 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2014 line_time.full = dfixed_div(line_time, a);
2015 bpp.full = dfixed_const(wm->bytes_per_pixel);
2016 src_width.full = dfixed_const(wm->src_width);
2017 bandwidth.full = dfixed_mul(src_width, bpp);
2018 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2019 bandwidth.full = dfixed_div(bandwidth, line_time);
2020
2021 return dfixed_trunc(bandwidth);
2022}
2023
2024static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2025{
2026 /* First calcualte the latency in ns */
2027 u32 mc_latency = 2000; /* 2000 ns. */
2028 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2029 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2030 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2031 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2032 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2033 (wm->num_heads * cursor_line_pair_return_time);
2034 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2035 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2036 fixed20_12 a, b, c;
2037
2038 if (wm->num_heads == 0)
2039 return 0;
2040
2041 a.full = dfixed_const(2);
2042 b.full = dfixed_const(1);
2043 if ((wm->vsc.full > a.full) ||
2044 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2045 (wm->vtaps >= 5) ||
2046 ((wm->vsc.full >= a.full) && wm->interlaced))
2047 max_src_lines_per_dst_line = 4;
2048 else
2049 max_src_lines_per_dst_line = 2;
2050
2051 a.full = dfixed_const(available_bandwidth);
2052 b.full = dfixed_const(wm->num_heads);
2053 a.full = dfixed_div(a, b);
2054
2055 b.full = dfixed_const(1000);
2056 c.full = dfixed_const(wm->disp_clk);
2057 b.full = dfixed_div(c, b);
2058 c.full = dfixed_const(wm->bytes_per_pixel);
2059 b.full = dfixed_mul(b, c);
2060
2061 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2062
2063 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2064 b.full = dfixed_const(1000);
2065 c.full = dfixed_const(lb_fill_bw);
2066 b.full = dfixed_div(c, b);
2067 a.full = dfixed_div(a, b);
2068 line_fill_time = dfixed_trunc(a);
2069
2070 if (line_fill_time < wm->active_time)
2071 return latency;
2072 else
2073 return latency + (line_fill_time - wm->active_time);
2074
2075}
2076
2077static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2078{
2079 if (evergreen_average_bandwidth(wm) <=
2080 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2081 return true;
2082 else
2083 return false;
2084};
2085
2086static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2087{
2088 if (evergreen_average_bandwidth(wm) <=
2089 (evergreen_available_bandwidth(wm) / wm->num_heads))
2090 return true;
2091 else
2092 return false;
2093};
2094
2095static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2096{
2097 u32 lb_partitions = wm->lb_size / wm->src_width;
2098 u32 line_time = wm->active_time + wm->blank_time;
2099 u32 latency_tolerant_lines;
2100 u32 latency_hiding;
2101 fixed20_12 a;
2102
2103 a.full = dfixed_const(1);
2104 if (wm->vsc.full > a.full)
2105 latency_tolerant_lines = 1;
2106 else {
2107 if (lb_partitions <= (wm->vtaps + 1))
2108 latency_tolerant_lines = 1;
2109 else
2110 latency_tolerant_lines = 2;
2111 }
2112
2113 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2114
2115 if (evergreen_latency_watermark(wm) <= latency_hiding)
2116 return true;
2117 else
2118 return false;
2119}
2120
2121static void evergreen_program_watermarks(struct radeon_device *rdev,
2122 struct radeon_crtc *radeon_crtc,
2123 u32 lb_size, u32 num_heads)
2124{
2125 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2126 struct evergreen_wm_params wm_low, wm_high;
2127 u32 dram_channels;
f9d9c362
AD		 2128 u32 pixel_period;
2129 u32 line_time = 0;
2130 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2131 u32 priority_a_mark = 0, priority_b_mark = 0;
2132 u32 priority_a_cnt = PRIORITY_OFF;
2133 u32 priority_b_cnt = PRIORITY_OFF;
2134 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2135 u32 tmp, arb_control3;
2136 fixed20_12 a, b, c;
2137
2138 if (radeon_crtc->base.enabled && num_heads && mode) {
2139 pixel_period = 1000000 / (u32)mode->clock;
2140 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2141 priority_a_cnt = 0;
2142 priority_b_cnt = 0;
cf0cfdd7
AD		 2143 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2144
2145 /* watermark for high clocks */
2146 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2147 wm_high.yclk =
2148 radeon_dpm_get_mclk(rdev, false) * 10;
2149 wm_high.sclk =
2150 radeon_dpm_get_sclk(rdev, false) * 10;
2151 } else {
2152 wm_high.yclk = rdev->pm.current_mclk * 10;
2153 wm_high.sclk = rdev->pm.current_sclk * 10;
2154 }
f9d9c362 2155
cf0cfdd7
AD		 2156 wm_high.disp_clk = mode->clock;
2157 wm_high.src_width = mode->crtc_hdisplay;
2158 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2159 wm_high.blank_time = line_time - wm_high.active_time;
2160 wm_high.interlaced = false;
f9d9c362 2161 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2162 wm_high.interlaced = true;
2163 wm_high.vsc = radeon_crtc->vsc;
2164 wm_high.vtaps = 1;
f9d9c362 2165 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2166 wm_high.vtaps = 2;
2167 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2168 wm_high.lb_size = lb_size;
2169 wm_high.dram_channels = dram_channels;
2170 wm_high.num_heads = num_heads;
2171
2172 /* watermark for low clocks */
2173 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2174 wm_low.yclk =
2175 radeon_dpm_get_mclk(rdev, true) * 10;
2176 wm_low.sclk =
2177 radeon_dpm_get_sclk(rdev, true) * 10;
2178 } else {
2179 wm_low.yclk = rdev->pm.current_mclk * 10;
2180 wm_low.sclk = rdev->pm.current_sclk * 10;
2181 }
2182
2183 wm_low.disp_clk = mode->clock;
2184 wm_low.src_width = mode->crtc_hdisplay;
2185 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2186 wm_low.blank_time = line_time - wm_low.active_time;
2187 wm_low.interlaced = false;
2188 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2189 wm_low.interlaced = true;
2190 wm_low.vsc = radeon_crtc->vsc;
2191 wm_low.vtaps = 1;
2192 if (radeon_crtc->rmx_type != RMX_OFF)
2193 wm_low.vtaps = 2;
2194 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2195 wm_low.lb_size = lb_size;
2196 wm_low.dram_channels = dram_channels;
2197 wm_low.num_heads = num_heads;
f9d9c362
AD		 2198
2199 /* set for high clocks */
cf0cfdd7 2200 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2201 /* set for low clocks */
cf0cfdd7 2202 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2203
2204 /* possibly force display priority to high */
2205 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2206 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2207 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2208 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2209 (rdev->disp_priority == 2)) {
cf0cfdd7 2210 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2211 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2212 }
2213 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2214 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2215 !evergreen_check_latency_hiding(&wm_low) ||
2216 (rdev->disp_priority == 2)) {
2217 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2218 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2219 }
2220
2221 a.full = dfixed_const(1000);
2222 b.full = dfixed_const(mode->clock);
2223 b.full = dfixed_div(b, a);
2224 c.full = dfixed_const(latency_watermark_a);
2225 c.full = dfixed_mul(c, b);
2226 c.full = dfixed_mul(c, radeon_crtc->hsc);
2227 c.full = dfixed_div(c, a);
2228 a.full = dfixed_const(16);
2229 c.full = dfixed_div(c, a);
2230 priority_a_mark = dfixed_trunc(c);
2231 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2232
2233 a.full = dfixed_const(1000);
2234 b.full = dfixed_const(mode->clock);
2235 b.full = dfixed_div(b, a);
2236 c.full = dfixed_const(latency_watermark_b);
2237 c.full = dfixed_mul(c, b);
2238 c.full = dfixed_mul(c, radeon_crtc->hsc);
2239 c.full = dfixed_div(c, a);
2240 a.full = dfixed_const(16);
2241 c.full = dfixed_div(c, a);
2242 priority_b_mark = dfixed_trunc(c);
2243 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2244 }
2245
2246 /* select wm A */
2247 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2248 tmp = arb_control3;
2249 tmp &= ~LATENCY_WATERMARK_MASK(3);
2250 tmp |= LATENCY_WATERMARK_MASK(1);
2251 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2252 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2253 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2254 LATENCY_HIGH_WATERMARK(line_time)));
2255 /* select wm B */
2256 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2257 tmp &= ~LATENCY_WATERMARK_MASK(3);
2258 tmp |= LATENCY_WATERMARK_MASK(2);
2259 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2260 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2261 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2262 LATENCY_HIGH_WATERMARK(line_time)));
2263 /* restore original selection */
2264 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2265
2266 /* write the priority marks */
2267 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2268 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2269
7178d2a6
AD		 2270 /* save values for DPM */
2271 radeon_crtc->line_time = line_time;
2272 radeon_crtc->wm_high = latency_watermark_a;
2273 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2274}
2275
377edc8b
AD		 2276/**
2277 * evergreen_bandwidth_update - update display watermarks callback.
2278 *
2279 * @rdev: radeon_device pointer
2280 *
2281 * Update the display watermarks based on the requested mode(s)
2282 * (evergreen+).
2283 */
0ca2ab52 2284void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2285{
f9d9c362
AD		 2286 struct drm_display_mode *mode0 = NULL;
2287 struct drm_display_mode *mode1 = NULL;
2288 u32 num_heads = 0, lb_size;
2289 int i;
2290
2291 radeon_update_display_priority(rdev);
2292
2293 for (i = 0; i < rdev->num_crtc; i++) {
2294 if (rdev->mode_info.crtcs[i]->base.enabled)
2295 num_heads++;
2296 }
2297 for (i = 0; i < rdev->num_crtc; i += 2) {
2298 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2299 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2300 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2301 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2302 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2303 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2304 }
bcc1c2a1
AD		 2305}
2306
377edc8b
AD		 2307/**
2308 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2309 *
2310 * @rdev: radeon_device pointer
2311 *
2312 * Wait for the MC (memory controller) to be idle.
2313 * (evergreen+).
2314 * Returns 0 if the MC is idle, -1 if not.
2315 */
b9952a8a 2316int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2317{
2318 unsigned i;
2319 u32 tmp;
2320
2321 for (i = 0; i < rdev->usec_timeout; i++) {
2322 /* read MC_STATUS */
2323 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2324 if (!tmp)
2325 return 0;
2326 udelay(1);
2327 }
2328 return -1;
2329}
2330
2331/*
2332 * GART
2333 */
0fcdb61e
AD		 2334void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2335{
2336 unsigned i;
2337 u32 tmp;
2338
6f2f48a9
AD		 2339 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2340
0fcdb61e
AD		 2341 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2342 for (i = 0; i < rdev->usec_timeout; i++) {
2343 /* read MC_STATUS */
2344 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2345 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2346 if (tmp == 2) {
2347 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2348 return;
2349 }
2350 if (tmp) {
2351 return;
2352 }
2353 udelay(1);
2354 }
2355}
2356
1109ca09 2357static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2358{
2359 u32 tmp;
0fcdb61e 2360 int r;
bcc1c2a1 2361
c9a1be96 2362 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2363 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2364 return -EINVAL;
2365 }
2366 r = radeon_gart_table_vram_pin(rdev);
2367 if (r)
2368 return r;
82568565 2369 radeon_gart_restore(rdev);
bcc1c2a1
AD		 2370 /* Setup L2 cache */
2371 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2372 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2373 EFFECTIVE_L2_QUEUE_SIZE(7));
2374 WREG32(VM_L2_CNTL2, 0);
2375 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2376 /* Setup TLB control */
2377 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2378 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2379 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2380 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2381 if (rdev->flags & RADEON_IS_IGP) {
2382 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2383 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2384 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2385 } else {
2386 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2387 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2388 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2389 if ((rdev->family == CHIP_JUNIPER) ||
2390 (rdev->family == CHIP_CYPRESS) ||
2391 (rdev->family == CHIP_HEMLOCK) ||
2392 (rdev->family == CHIP_BARTS))
2393 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2394 }
bcc1c2a1
AD		 2395 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2396 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2397 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2398 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2399 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2400 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2401 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2402 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2403 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2404 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2405 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2406 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2407
0fcdb61e 2408 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2409 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2410 (unsigned)(rdev->mc.gtt_size >> 20),
2411 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2412 rdev->gart.ready = true;
2413 return 0;
2414}
2415
1109ca09 2416static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2417{
2418 u32 tmp;
bcc1c2a1
AD		 2419
2420 /* Disable all tables */
0fcdb61e
AD		 2421 WREG32(VM_CONTEXT0_CNTL, 0);
2422 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2423
2424 /* Setup L2 cache */
2425 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2426 EFFECTIVE_L2_QUEUE_SIZE(7));
2427 WREG32(VM_L2_CNTL2, 0);
2428 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2429 /* Setup TLB control */
2430 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2431 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2432 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2433 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2434 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2435 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2436 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2437 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2438 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2439}
2440
1109ca09 2441static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2442{
2443 evergreen_pcie_gart_disable(rdev);
2444 radeon_gart_table_vram_free(rdev);
2445 radeon_gart_fini(rdev);
2446}
2447
2448
1109ca09 2449static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2450{
2451 u32 tmp;
bcc1c2a1
AD		 2452
2453 /* Setup L2 cache */
2454 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2455 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2456 EFFECTIVE_L2_QUEUE_SIZE(7));
2457 WREG32(VM_L2_CNTL2, 0);
2458 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2459 /* Setup TLB control */
2460 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2461 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2462 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2463 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2464 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2465 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2466 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2467 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2468 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2469 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2470 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2471 WREG32(VM_CONTEXT0_CNTL, 0);
2472 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2473}
2474
b9952a8a 2475void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2476{
62444b74
AD		 2477 u32 crtc_enabled, tmp, frame_count, blackout;
2478 int i, j;
2479
5153550a
AD		 2480 if (!ASIC_IS_NODCE(rdev)) {
2481 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2482 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2483
5153550a
AD		 2484 /* disable VGA render */
2485 WREG32(VGA_RENDER_CONTROL, 0);
2486 }
62444b74
AD		 2487 /* blank the display controllers */
2488 for (i = 0; i < rdev->num_crtc; i++) {
2489 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2490 if (crtc_enabled) {
2491 save->crtc_enabled[i] = true;
2492 if (ASIC_IS_DCE6(rdev)) {
2493 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2494 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2495 radeon_wait_for_vblank(rdev, i);
abf1457b 2496 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2497 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2498 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2499 }
2500 } else {
2501 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2502 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2503 radeon_wait_for_vblank(rdev, i);
abf1457b 2504 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2505 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2506 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2507 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2508 }
2509 }
2510 /* wait for the next frame */
2511 frame_count = radeon_get_vblank_counter(rdev, i);
2512 for (j = 0; j < rdev->usec_timeout; j++) {
2513 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2514 break;
2515 udelay(1);
2516 }
abf1457b
AD		 2517
2518 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2519 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2520 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2521 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2522 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2523 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2524 save->crtc_enabled[i] = false;
2525 /* ***** */
804cc4a0
AD		 2526 } else {
2527 save->crtc_enabled[i] = false;
62444b74 2528 }
18007401 2529 }
bcc1c2a1 2530
62444b74
AD		 2531 radeon_mc_wait_for_idle(rdev);
2532
2533 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2534 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2535 /* Block CPU access */
2536 WREG32(BIF_FB_EN, 0);
2537 /* blackout the MC */
2538 blackout &= ~BLACKOUT_MODE_MASK;
2539 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2540 }
ed39fadd
AD		 2541 /* wait for the MC to settle */
2542 udelay(100);
968c0166
AD		 2543
2544 /* lock double buffered regs */
2545 for (i = 0; i < rdev->num_crtc; i++) {
2546 if (save->crtc_enabled[i]) {
2547 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2548 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2549 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2550 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2551 }
2552 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2553 if (!(tmp & 1)) {
2554 tmp |= 1;
2555 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2556 }
2557 }
2558 }
bcc1c2a1
AD		 2559}
2560
b9952a8a 2561void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2562{
62444b74
AD		 2563 u32 tmp, frame_count;
2564 int i, j;
18007401 2565
62444b74
AD		 2566 /* update crtc base addresses */
2567 for (i = 0; i < rdev->num_crtc; i++) {
2568 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2569 upper_32_bits(rdev->mc.vram_start));
62444b74 2570 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2571 upper_32_bits(rdev->mc.vram_start));
62444b74 2572 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2573 (u32)rdev->mc.vram_start);
62444b74 2574 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2575 (u32)rdev->mc.vram_start);
2576 }
5153550a
AD		 2577
2578 if (!ASIC_IS_NODCE(rdev)) {
2579 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2580 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2581 }
62444b74 2582
968c0166
AD		 2583 /* unlock regs and wait for update */
2584 for (i = 0; i < rdev->num_crtc; i++) {
2585 if (save->crtc_enabled[i]) {
2586 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2587 if ((tmp & 0x3) != 0) {
2588 tmp &= ~0x3;
2589 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2590 }
2591 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2592 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2593 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2594 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2595 }
2596 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2597 if (tmp & 1) {
2598 tmp &= ~1;
2599 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2600 }
2601 for (j = 0; j < rdev->usec_timeout; j++) {
2602 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2603 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2604 break;
2605 udelay(1);
2606 }
2607 }
2608 }
2609
62444b74
AD		 2610 /* unblackout the MC */
2611 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2612 tmp &= ~BLACKOUT_MODE_MASK;
2613 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2614 /* allow CPU access */
2615 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2616
2617 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2618 if (save->crtc_enabled[i]) {
62444b74
AD		 2619 if (ASIC_IS_DCE6(rdev)) {
2620 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2621 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2622 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2623 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2624 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2625 } else {
2626 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2627 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2628 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2629 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2630 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2631 }
2632 /* wait for the next frame */
2633 frame_count = radeon_get_vblank_counter(rdev, i);
2634 for (j = 0; j < rdev->usec_timeout; j++) {
2635 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2636 break;
2637 udelay(1);
2638 }
2639 }
2640 }
5153550a
AD		 2641 if (!ASIC_IS_NODCE(rdev)) {
2642 /* Unlock vga access */
2643 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2644 mdelay(1);
2645 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2646 }
bcc1c2a1
AD		 2647}
2648
755d819e 2649void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2650{
2651 struct evergreen_mc_save save;
2652 u32 tmp;
2653 int i, j;
2654
2655 /* Initialize HDP */
2656 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2657 WREG32((0x2c14 + j), 0x00000000);
2658 WREG32((0x2c18 + j), 0x00000000);
2659 WREG32((0x2c1c + j), 0x00000000);
2660 WREG32((0x2c20 + j), 0x00000000);
2661 WREG32((0x2c24 + j), 0x00000000);
2662 }
2663 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2664
2665 evergreen_mc_stop(rdev, &save);
2666 if (evergreen_mc_wait_for_idle(rdev)) {
2667 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2668 }
2669 /* Lockout access through VGA aperture*/
2670 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2671 /* Update configuration */
2672 if (rdev->flags & RADEON_IS_AGP) {
2673 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2674 /* VRAM before AGP */
2675 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2676 rdev->mc.vram_start >> 12);
2677 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2678 rdev->mc.gtt_end >> 12);
2679 } else {
2680 /* VRAM after AGP */
2681 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2682 rdev->mc.gtt_start >> 12);
2683 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2684 rdev->mc.vram_end >> 12);
2685 }
2686 } else {
2687 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2688 rdev->mc.vram_start >> 12);
2689 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2690 rdev->mc.vram_end >> 12);
2691 }
3b9832f6 2692 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2693 /* llano/ontario only */
2694 if ((rdev->family == CHIP_PALM) ||
2695 (rdev->family == CHIP_SUMO) ||
2696 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2697 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2698 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2699 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2700 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2701 }
bcc1c2a1
AD		 2702 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2703 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2704 WREG32(MC_VM_FB_LOCATION, tmp);
2705 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2706 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2707 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2708 if (rdev->flags & RADEON_IS_AGP) {
2709 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2710 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2711 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2712 } else {
2713 WREG32(MC_VM_AGP_BASE, 0);
2714 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2715 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2716 }
2717 if (evergreen_mc_wait_for_idle(rdev)) {
2718 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2719 }
2720 evergreen_mc_resume(rdev, &save);
2721 /* we need to own VRAM, so turn off the VGA renderer here
2722 * to stop it overwriting our objects */
2723 rv515_vga_render_disable(rdev);
2724}
2725
bcc1c2a1
AD		 2726/*
2727 * CP.
2728 */
12920591
AD		 2729void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2730{
876dc9f3 2731 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2732 u32 next_rptr;
7b1f2485 2733
12920591 2734 /* set to DX10/11 mode */
e32eb50d
CK		 2735 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2736 radeon_ring_write(ring, 1);
45df6803
CK		 2737
2738 if (ring->rptr_save_reg) {
89d35807 2739 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2740 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2741 radeon_ring_write(ring, ((ring->rptr_save_reg -
2742 PACKET3_SET_CONFIG_REG_START) >> 2));
2743 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2744 } else if (rdev->wb.enabled) {
2745 next_rptr = ring->wptr + 5 + 4;
2746 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2747 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2748 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2749 radeon_ring_write(ring, next_rptr);
2750 radeon_ring_write(ring, 0);
45df6803
CK		 2751 }
2752
e32eb50d
CK		 2753 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2754 radeon_ring_write(ring,
0f234f5f
AD		 2755#ifdef __BIG_ENDIAN
2756 (2 << 0) |
2757#endif
2758 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2759 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2760 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2761}
2762
bcc1c2a1
AD		 2763
2764static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2765{
fe251e2f
AD		 2766 const __be32 *fw_data;
2767 int i;
2768
2769 if (!rdev->me_fw || !rdev->pfp_fw)
2770 return -EINVAL;
bcc1c2a1 2771
fe251e2f 2772 r700_cp_stop(rdev);
0f234f5f
AD		 2773 WREG32(CP_RB_CNTL,
2774#ifdef __BIG_ENDIAN
2775 BUF_SWAP_32BIT |
2776#endif
2777 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2778
2779 fw_data = (const __be32 *)rdev->pfp_fw->data;
2780 WREG32(CP_PFP_UCODE_ADDR, 0);
2781 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2782 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2783 WREG32(CP_PFP_UCODE_ADDR, 0);
2784
2785 fw_data = (const __be32 *)rdev->me_fw->data;
2786 WREG32(CP_ME_RAM_WADDR, 0);
2787 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2788 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2789
2790 WREG32(CP_PFP_UCODE_ADDR, 0);
2791 WREG32(CP_ME_RAM_WADDR, 0);
2792 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2793 return 0;
2794}
2795
7e7b41d2
AD		 2796static int evergreen_cp_start(struct radeon_device *rdev)
2797{
e32eb50d 2798 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2799 int r, i;
7e7b41d2
AD		 2800 uint32_t cp_me;
2801
e32eb50d 2802 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2803 if (r) {
2804 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2805 return r;
2806 }
e32eb50d
CK		 2807 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2808 radeon_ring_write(ring, 0x1);
2809 radeon_ring_write(ring, 0x0);
2810 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2811 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2812 radeon_ring_write(ring, 0);
2813 radeon_ring_write(ring, 0);
2814 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2815
2816 cp_me = 0xff;
2817 WREG32(CP_ME_CNTL, cp_me);
2818
e32eb50d 2819 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2820 if (r) {
2821 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2822 return r;
2823 }
2281a378
AD		 2824
2825 /* setup clear context state */
e32eb50d
CK		 2826 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2827 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2828
2829 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2830 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2831
e32eb50d
CK		 2832 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2833 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2834
2835 /* set clear context state */
e32eb50d
CK		 2836 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2837 radeon_ring_write(ring, 0);
2281a378
AD		 2838
2839 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2840 radeon_ring_write(ring, 0xc0026f00);
2841 radeon_ring_write(ring, 0x00000000);
2842 radeon_ring_write(ring, 0x00000000);
2843 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2844
2845 /* Clear consts */
e32eb50d
CK		 2846 radeon_ring_write(ring, 0xc0036f00);
2847 radeon_ring_write(ring, 0x00000bc4);
2848 radeon_ring_write(ring, 0xffffffff);
2849 radeon_ring_write(ring, 0xffffffff);
2850 radeon_ring_write(ring, 0xffffffff);
2281a378 2851
e32eb50d
CK		 2852 radeon_ring_write(ring, 0xc0026900);
2853 radeon_ring_write(ring, 0x00000316);
2854 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2855 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2856
e32eb50d 2857 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2858
2859 return 0;
2860}
2861
1109ca09 2862static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2863{
e32eb50d 2864 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2865 u32 tmp;
2866 u32 rb_bufsz;
2867 int r;
2868
2869 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2870 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2871 SOFT_RESET_PA |
2872 SOFT_RESET_SH |
2873 SOFT_RESET_VGT |
a49a50da 2874 SOFT_RESET_SPI |
fe251e2f
AD		 2875 SOFT_RESET_SX));
2876 RREG32(GRBM_SOFT_RESET);
2877 mdelay(15);
2878 WREG32(GRBM_SOFT_RESET, 0);
2879 RREG32(GRBM_SOFT_RESET);
2880
2881 /* Set ring buffer size */
e32eb50d 2882 rb_bufsz = drm_order(ring->ring_size / 8);
724c80e1 2883 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2884#ifdef __BIG_ENDIAN
2885 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2886#endif
fe251e2f 2887 WREG32(CP_RB_CNTL, tmp);
15d3332f 2888 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2889 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2890
2891 /* Set the write pointer delay */
2892 WREG32(CP_RB_WPTR_DELAY, 0);
2893
2894 /* Initialize the ring buffer's read and write pointers */
2895 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2896 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2897 ring->wptr = 0;
2898 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2899
48fc7f7e 2900 /* set the wb address whether it's enabled or not */
0f234f5f 2901 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2902 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2903 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2904 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2905
2906 if (rdev->wb.enabled)
2907 WREG32(SCRATCH_UMSK, 0xff);
2908 else {
2909 tmp |= RB_NO_UPDATE;
2910 WREG32(SCRATCH_UMSK, 0);
2911 }
2912
fe251e2f
AD		 2913 mdelay(1);
2914 WREG32(CP_RB_CNTL, tmp);
2915
e32eb50d 2916 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2917 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2918
e32eb50d 2919 ring->rptr = RREG32(CP_RB_RPTR);
fe251e2f 2920
7e7b41d2 2921 evergreen_cp_start(rdev);
e32eb50d 2922 ring->ready = true;
f712812e 2923 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 2924 if (r) {
e32eb50d 2925 ring->ready = false;
fe251e2f
AD		 2926 return r;
2927 }
2928 return 0;
2929}
bcc1c2a1
AD		 2930
2931/*
2932 * Core functions
2933 */
bcc1c2a1
AD		 2934static void evergreen_gpu_init(struct radeon_device *rdev)
2935{
416a2bd2 2936 u32 gb_addr_config;
32fcdbf4 2937 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 2938 u32 sx_debug_1;
2939 u32 smx_dc_ctl0;
2940 u32 sq_config;
2941 u32 sq_lds_resource_mgmt;
2942 u32 sq_gpr_resource_mgmt_1;
2943 u32 sq_gpr_resource_mgmt_2;
2944 u32 sq_gpr_resource_mgmt_3;
2945 u32 sq_thread_resource_mgmt;
2946 u32 sq_thread_resource_mgmt_2;
2947 u32 sq_stack_resource_mgmt_1;
2948 u32 sq_stack_resource_mgmt_2;
2949 u32 sq_stack_resource_mgmt_3;
2950 u32 vgt_cache_invalidation;
f25a5c63 2951 u32 hdp_host_path_cntl, tmp;
416a2bd2 2952 u32 disabled_rb_mask;
32fcdbf4
AD		 2953 int i, j, num_shader_engines, ps_thread_count;
2954
2955 switch (rdev->family) {
2956 case CHIP_CYPRESS:
2957 case CHIP_HEMLOCK:
2958 rdev->config.evergreen.num_ses = 2;
2959 rdev->config.evergreen.max_pipes = 4;
2960 rdev->config.evergreen.max_tile_pipes = 8;
2961 rdev->config.evergreen.max_simds = 10;
2962 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2963 rdev->config.evergreen.max_gprs = 256;
2964 rdev->config.evergreen.max_threads = 248;
2965 rdev->config.evergreen.max_gs_threads = 32;
2966 rdev->config.evergreen.max_stack_entries = 512;
2967 rdev->config.evergreen.sx_num_of_sets = 4;
2968 rdev->config.evergreen.sx_max_export_size = 256;
2969 rdev->config.evergreen.sx_max_export_pos_size = 64;
2970 rdev->config.evergreen.sx_max_export_smx_size = 192;
2971 rdev->config.evergreen.max_hw_contexts = 8;
2972 rdev->config.evergreen.sq_num_cf_insts = 2;
2973
2974 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2975 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2976 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2977 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2978 break;
2979 case CHIP_JUNIPER:
2980 rdev->config.evergreen.num_ses = 1;
2981 rdev->config.evergreen.max_pipes = 4;
2982 rdev->config.evergreen.max_tile_pipes = 4;
2983 rdev->config.evergreen.max_simds = 10;
2984 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2985 rdev->config.evergreen.max_gprs = 256;
2986 rdev->config.evergreen.max_threads = 248;
2987 rdev->config.evergreen.max_gs_threads = 32;
2988 rdev->config.evergreen.max_stack_entries = 512;
2989 rdev->config.evergreen.sx_num_of_sets = 4;
2990 rdev->config.evergreen.sx_max_export_size = 256;
2991 rdev->config.evergreen.sx_max_export_pos_size = 64;
2992 rdev->config.evergreen.sx_max_export_smx_size = 192;
2993 rdev->config.evergreen.max_hw_contexts = 8;
2994 rdev->config.evergreen.sq_num_cf_insts = 2;
2995
2996 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2997 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2998 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2999 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3000 break;
3001 case CHIP_REDWOOD:
3002 rdev->config.evergreen.num_ses = 1;
3003 rdev->config.evergreen.max_pipes = 4;
3004 rdev->config.evergreen.max_tile_pipes = 4;
3005 rdev->config.evergreen.max_simds = 5;
3006 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3007 rdev->config.evergreen.max_gprs = 256;
3008 rdev->config.evergreen.max_threads = 248;
3009 rdev->config.evergreen.max_gs_threads = 32;
3010 rdev->config.evergreen.max_stack_entries = 256;
3011 rdev->config.evergreen.sx_num_of_sets = 4;
3012 rdev->config.evergreen.sx_max_export_size = 256;
3013 rdev->config.evergreen.sx_max_export_pos_size = 64;
3014 rdev->config.evergreen.sx_max_export_smx_size = 192;
3015 rdev->config.evergreen.max_hw_contexts = 8;
3016 rdev->config.evergreen.sq_num_cf_insts = 2;
3017
3018 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3019 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3020 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3021 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3022 break;
3023 case CHIP_CEDAR:
3024 default:
3025 rdev->config.evergreen.num_ses = 1;
3026 rdev->config.evergreen.max_pipes = 2;
3027 rdev->config.evergreen.max_tile_pipes = 2;
3028 rdev->config.evergreen.max_simds = 2;
3029 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3030 rdev->config.evergreen.max_gprs = 256;
3031 rdev->config.evergreen.max_threads = 192;
3032 rdev->config.evergreen.max_gs_threads = 16;
3033 rdev->config.evergreen.max_stack_entries = 256;
3034 rdev->config.evergreen.sx_num_of_sets = 4;
3035 rdev->config.evergreen.sx_max_export_size = 128;
3036 rdev->config.evergreen.sx_max_export_pos_size = 32;
3037 rdev->config.evergreen.sx_max_export_smx_size = 96;
3038 rdev->config.evergreen.max_hw_contexts = 4;
3039 rdev->config.evergreen.sq_num_cf_insts = 1;
3040
d5e455e4
AD		 3041 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3042 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3043 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3044 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3045 break;
3046 case CHIP_PALM:
3047 rdev->config.evergreen.num_ses = 1;
3048 rdev->config.evergreen.max_pipes = 2;
3049 rdev->config.evergreen.max_tile_pipes = 2;
3050 rdev->config.evergreen.max_simds = 2;
3051 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3052 rdev->config.evergreen.max_gprs = 256;
3053 rdev->config.evergreen.max_threads = 192;
3054 rdev->config.evergreen.max_gs_threads = 16;
3055 rdev->config.evergreen.max_stack_entries = 256;
3056 rdev->config.evergreen.sx_num_of_sets = 4;
3057 rdev->config.evergreen.sx_max_export_size = 128;
3058 rdev->config.evergreen.sx_max_export_pos_size = 32;
3059 rdev->config.evergreen.sx_max_export_smx_size = 96;
3060 rdev->config.evergreen.max_hw_contexts = 4;
3061 rdev->config.evergreen.sq_num_cf_insts = 1;
3062
d5c5a72f
AD		 3063 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3064 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3065 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3066 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3067 break;
3068 case CHIP_SUMO:
3069 rdev->config.evergreen.num_ses = 1;
3070 rdev->config.evergreen.max_pipes = 4;
bd25f078 3071 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3072 if (rdev->pdev->device == 0x9648)
3073 rdev->config.evergreen.max_simds = 3;
3074 else if ((rdev->pdev->device == 0x9647) ||
3075 (rdev->pdev->device == 0x964a))
3076 rdev->config.evergreen.max_simds = 4;
3077 else
3078 rdev->config.evergreen.max_simds = 5;
3079 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3080 rdev->config.evergreen.max_gprs = 256;
3081 rdev->config.evergreen.max_threads = 248;
3082 rdev->config.evergreen.max_gs_threads = 32;
3083 rdev->config.evergreen.max_stack_entries = 256;
3084 rdev->config.evergreen.sx_num_of_sets = 4;
3085 rdev->config.evergreen.sx_max_export_size = 256;
3086 rdev->config.evergreen.sx_max_export_pos_size = 64;
3087 rdev->config.evergreen.sx_max_export_smx_size = 192;
3088 rdev->config.evergreen.max_hw_contexts = 8;
3089 rdev->config.evergreen.sq_num_cf_insts = 2;
3090
3091 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3092 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3093 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3094 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3095 break;
3096 case CHIP_SUMO2:
3097 rdev->config.evergreen.num_ses = 1;
3098 rdev->config.evergreen.max_pipes = 4;
3099 rdev->config.evergreen.max_tile_pipes = 4;
3100 rdev->config.evergreen.max_simds = 2;
3101 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3102 rdev->config.evergreen.max_gprs = 256;
3103 rdev->config.evergreen.max_threads = 248;
3104 rdev->config.evergreen.max_gs_threads = 32;
3105 rdev->config.evergreen.max_stack_entries = 512;
3106 rdev->config.evergreen.sx_num_of_sets = 4;
3107 rdev->config.evergreen.sx_max_export_size = 256;
3108 rdev->config.evergreen.sx_max_export_pos_size = 64;
3109 rdev->config.evergreen.sx_max_export_smx_size = 192;
3110 rdev->config.evergreen.max_hw_contexts = 8;
3111 rdev->config.evergreen.sq_num_cf_insts = 2;
3112
adb68fa2
AD		 3113 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3114 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3115 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3116 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3117 break;
3118 case CHIP_BARTS:
3119 rdev->config.evergreen.num_ses = 2;
3120 rdev->config.evergreen.max_pipes = 4;
3121 rdev->config.evergreen.max_tile_pipes = 8;
3122 rdev->config.evergreen.max_simds = 7;
3123 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3124 rdev->config.evergreen.max_gprs = 256;
3125 rdev->config.evergreen.max_threads = 248;
3126 rdev->config.evergreen.max_gs_threads = 32;
3127 rdev->config.evergreen.max_stack_entries = 512;
3128 rdev->config.evergreen.sx_num_of_sets = 4;
3129 rdev->config.evergreen.sx_max_export_size = 256;
3130 rdev->config.evergreen.sx_max_export_pos_size = 64;
3131 rdev->config.evergreen.sx_max_export_smx_size = 192;
3132 rdev->config.evergreen.max_hw_contexts = 8;
3133 rdev->config.evergreen.sq_num_cf_insts = 2;
3134
3135 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3136 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3137 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3138 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3139 break;
3140 case CHIP_TURKS:
3141 rdev->config.evergreen.num_ses = 1;
3142 rdev->config.evergreen.max_pipes = 4;
3143 rdev->config.evergreen.max_tile_pipes = 4;
3144 rdev->config.evergreen.max_simds = 6;
3145 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3146 rdev->config.evergreen.max_gprs = 256;
3147 rdev->config.evergreen.max_threads = 248;
3148 rdev->config.evergreen.max_gs_threads = 32;
3149 rdev->config.evergreen.max_stack_entries = 256;
3150 rdev->config.evergreen.sx_num_of_sets = 4;
3151 rdev->config.evergreen.sx_max_export_size = 256;
3152 rdev->config.evergreen.sx_max_export_pos_size = 64;
3153 rdev->config.evergreen.sx_max_export_smx_size = 192;
3154 rdev->config.evergreen.max_hw_contexts = 8;
3155 rdev->config.evergreen.sq_num_cf_insts = 2;
3156
3157 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3158 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3159 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3160 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3161 break;
3162 case CHIP_CAICOS:
3163 rdev->config.evergreen.num_ses = 1;
bd25f078 3164 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3165 rdev->config.evergreen.max_tile_pipes = 2;
3166 rdev->config.evergreen.max_simds = 2;
3167 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3168 rdev->config.evergreen.max_gprs = 256;
3169 rdev->config.evergreen.max_threads = 192;
3170 rdev->config.evergreen.max_gs_threads = 16;
3171 rdev->config.evergreen.max_stack_entries = 256;
3172 rdev->config.evergreen.sx_num_of_sets = 4;
3173 rdev->config.evergreen.sx_max_export_size = 128;
3174 rdev->config.evergreen.sx_max_export_pos_size = 32;
3175 rdev->config.evergreen.sx_max_export_smx_size = 96;
3176 rdev->config.evergreen.max_hw_contexts = 4;
3177 rdev->config.evergreen.sq_num_cf_insts = 1;
3178
32fcdbf4
AD		 3179 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3180 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3181 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3182 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3183 break;
3184 }
3185
3186 /* Initialize HDP */
3187 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3188 WREG32((0x2c14 + j), 0x00000000);
3189 WREG32((0x2c18 + j), 0x00000000);
3190 WREG32((0x2c1c + j), 0x00000000);
3191 WREG32((0x2c20 + j), 0x00000000);
3192 WREG32((0x2c24 + j), 0x00000000);
3193 }
3194
3195 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3196
d054ac16
AD		 3197 evergreen_fix_pci_max_read_req_size(rdev);
3198
32fcdbf4 3199 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3200 if ((rdev->family == CHIP_PALM) ||
3201 (rdev->family == CHIP_SUMO) ||
3202 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3203 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3204 else
3205 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3206
1aa52bd3
AD		 3207 /* setup tiling info dword. gb_addr_config is not adequate since it does
3208 * not have bank info, so create a custom tiling dword.
3209 * bits 3:0 num_pipes
3210 * bits 7:4 num_banks
3211 * bits 11:8 group_size
3212 * bits 15:12 row_size
3213 */
3214 rdev->config.evergreen.tile_config = 0;
3215 switch (rdev->config.evergreen.max_tile_pipes) {
3216 case 1:
3217 default:
3218 rdev->config.evergreen.tile_config |= (0 << 0);
3219 break;
3220 case 2:
3221 rdev->config.evergreen.tile_config |= (1 << 0);
3222 break;
3223 case 4:
3224 rdev->config.evergreen.tile_config |= (2 << 0);
3225 break;
3226 case 8:
3227 rdev->config.evergreen.tile_config |= (3 << 0);
3228 break;
3229 }
d698a34d 3230 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3231 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3232 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3233 else {
c8d15edc
AD		 3234 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3235 case 0: /* four banks */
29d65406 3236 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3237 break;
3238 case 1: /* eight banks */
3239 rdev->config.evergreen.tile_config |= 1 << 4;
3240 break;
3241 case 2: /* sixteen banks */
3242 default:
3243 rdev->config.evergreen.tile_config |= 2 << 4;
3244 break;
3245 }
29d65406 3246 }
416a2bd2 3247 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3248 rdev->config.evergreen.tile_config |=
3249 ((gb_addr_config & 0x30000000) >> 28) << 12;
3250
416a2bd2 3251 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
32fcdbf4 3252
416a2bd2
AD		 3253 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3254 u32 efuse_straps_4;
3255 u32 efuse_straps_3;
32fcdbf4 3256
ff82bbc4
AD		 3257 efuse_straps_4 = RREG32_RCU(0x204);
3258 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3259 tmp = (((efuse_straps_4 & 0xf) << 4) |
3260 ((efuse_straps_3 & 0xf0000000) >> 28));
3261 } else {
3262 tmp = 0;
3263 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3264 u32 rb_disable_bitmap;
3265
3266 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3267 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3268 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3269 tmp <<= 4;
3270 tmp |= rb_disable_bitmap;
32fcdbf4 3271 }
416a2bd2
AD		 3272 }
3273 /* enabled rb are just the one not disabled :) */
3274 disabled_rb_mask = tmp;
cedb655a
AD		 3275 tmp = 0;
3276 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3277 tmp |= (1 << i);
3278 /* if all the backends are disabled, fix it up here */
3279 if ((disabled_rb_mask & tmp) == tmp) {
3280 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3281 disabled_rb_mask &= ~(1 << i);
3282 }
32fcdbf4 3283
416a2bd2
AD		 3284 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3285 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3286
416a2bd2
AD		 3287 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3288 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3289 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3290 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3291 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3292 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3293 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3294
f7eb9730
AD		 3295 if ((rdev->config.evergreen.max_backends == 1) &&
3296 (rdev->flags & RADEON_IS_IGP)) {
3297 if ((disabled_rb_mask & 3) == 1) {
3298 /* RB0 disabled, RB1 enabled */
3299 tmp = 0x11111111;
3300 } else {
3301 /* RB1 disabled, RB0 enabled */
3302 tmp = 0x00000000;
3303 }
3304 } else {
3305 tmp = gb_addr_config & NUM_PIPES_MASK;
3306 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3307 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3308 }
416a2bd2 3309 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3310
3311 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3312 WREG32(CGTS_TCC_DISABLE, 0);
3313 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3314 WREG32(CGTS_USER_TCC_DISABLE, 0);
3315
3316 /* set HW defaults for 3D engine */
3317 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3318 ROQ_IB2_START(0x2b)));
3319
3320 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3321
3322 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3323 SYNC_GRADIENT |
3324 SYNC_WALKER |
3325 SYNC_ALIGNER));
3326
3327 sx_debug_1 = RREG32(SX_DEBUG_1);
3328 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3329 WREG32(SX_DEBUG_1, sx_debug_1);
3330
3331
3332 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3333 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3334 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3335 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3336
b866d133
AD		 3337 if (rdev->family <= CHIP_SUMO2)
3338 WREG32(SMX_SAR_CTL0, 0x00010000);
3339
32fcdbf4
AD		 3340 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3341 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3342 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3343
3344 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3345 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3346 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3347
3348 WREG32(VGT_NUM_INSTANCES, 1);
3349 WREG32(SPI_CONFIG_CNTL, 0);
3350 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3351 WREG32(CP_PERFMON_CNTL, 0);
3352
3353 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3354 FETCH_FIFO_HIWATER(0x4) |
3355 DONE_FIFO_HIWATER(0xe0) |
3356 ALU_UPDATE_FIFO_HIWATER(0x8)));
3357
3358 sq_config = RREG32(SQ_CONFIG);
3359 sq_config &= ~(PS_PRIO(3) |
3360 VS_PRIO(3) |
3361 GS_PRIO(3) |
3362 ES_PRIO(3));
3363 sq_config |= (VC_ENABLE |
3364 EXPORT_SRC_C |
3365 PS_PRIO(0) |
3366 VS_PRIO(1) |
3367 GS_PRIO(2) |
3368 ES_PRIO(3));
3369
d5e455e4
AD		 3370 switch (rdev->family) {
3371 case CHIP_CEDAR:
3372 case CHIP_PALM:
d5c5a72f
AD		 3373 case CHIP_SUMO:
3374 case CHIP_SUMO2:
adb68fa2 3375 case CHIP_CAICOS:
32fcdbf4
AD		 3376 /* no vertex cache */
3377 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3378 break;
3379 default:
3380 break;
3381 }
32fcdbf4
AD		 3382
3383 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3384
3385 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3386 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3387 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3388 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3389 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3390 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3391 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3392
d5e455e4
AD		 3393 switch (rdev->family) {
3394 case CHIP_CEDAR:
3395 case CHIP_PALM:
d5c5a72f
AD		 3396 case CHIP_SUMO:
3397 case CHIP_SUMO2:
32fcdbf4 3398 ps_thread_count = 96;
d5e455e4
AD		 3399 break;
3400 default:
32fcdbf4 3401 ps_thread_count = 128;
d5e455e4
AD		 3402 break;
3403 }
32fcdbf4
AD		 3404
3405 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3406 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3407 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3408 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3409 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3410 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3411
3412 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3413 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3414 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3415 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3416 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3417 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3418
3419 WREG32(SQ_CONFIG, sq_config);
3420 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3421 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3422 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3423 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3424 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3425 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3426 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3427 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3428 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3429 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3430
3431 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3432 FORCE_EOV_MAX_REZ_CNT(255)));
3433
d5e455e4
AD		 3434 switch (rdev->family) {
3435 case CHIP_CEDAR:
3436 case CHIP_PALM:
d5c5a72f
AD		 3437 case CHIP_SUMO:
3438 case CHIP_SUMO2:
adb68fa2 3439 case CHIP_CAICOS:
32fcdbf4 3440 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3441 break;
3442 default:
32fcdbf4 3443 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3444 break;
3445 }
32fcdbf4
AD		 3446 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3447 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3448
3449 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3450 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3451 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3452
60a4a3e0
AD		 3453 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3454 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3455
32fcdbf4
AD		 3456 WREG32(CB_PERF_CTR0_SEL_0, 0);
3457 WREG32(CB_PERF_CTR0_SEL_1, 0);
3458 WREG32(CB_PERF_CTR1_SEL_0, 0);
3459 WREG32(CB_PERF_CTR1_SEL_1, 0);
3460 WREG32(CB_PERF_CTR2_SEL_0, 0);
3461 WREG32(CB_PERF_CTR2_SEL_1, 0);
3462 WREG32(CB_PERF_CTR3_SEL_0, 0);
3463 WREG32(CB_PERF_CTR3_SEL_1, 0);
3464
60a4a3e0
AD		 3465 /* clear render buffer base addresses */
3466 WREG32(CB_COLOR0_BASE, 0);
3467 WREG32(CB_COLOR1_BASE, 0);
3468 WREG32(CB_COLOR2_BASE, 0);
3469 WREG32(CB_COLOR3_BASE, 0);
3470 WREG32(CB_COLOR4_BASE, 0);
3471 WREG32(CB_COLOR5_BASE, 0);
3472 WREG32(CB_COLOR6_BASE, 0);
3473 WREG32(CB_COLOR7_BASE, 0);
3474 WREG32(CB_COLOR8_BASE, 0);
3475 WREG32(CB_COLOR9_BASE, 0);
3476 WREG32(CB_COLOR10_BASE, 0);
3477 WREG32(CB_COLOR11_BASE, 0);
3478
3479 /* set the shader const cache sizes to 0 */
3480 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3481 WREG32(i, 0);
3482 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3483 WREG32(i, 0);
3484
f25a5c63
AD		 3485 tmp = RREG32(HDP_MISC_CNTL);
3486 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3487 WREG32(HDP_MISC_CNTL, tmp);
3488
32fcdbf4
AD		 3489 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3490 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3491
3492 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3493
3494 udelay(50);
3495
bcc1c2a1
AD		 3496}
3497
3498int evergreen_mc_init(struct radeon_device *rdev)
3499{
bcc1c2a1
AD		 3500 u32 tmp;
3501 int chansize, numchan;
bcc1c2a1
AD		 3502
3503 /* Get VRAM informations */
3504 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3505 if ((rdev->family == CHIP_PALM) ||
3506 (rdev->family == CHIP_SUMO) ||
3507 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3508 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3509 else
3510 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3511 if (tmp & CHANSIZE_OVERRIDE) {
3512 chansize = 16;
3513 } else if (tmp & CHANSIZE_MASK) {
3514 chansize = 64;
3515 } else {
3516 chansize = 32;
3517 }
3518 tmp = RREG32(MC_SHARED_CHMAP);
3519 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3520 case 0:
3521 default:
3522 numchan = 1;
3523 break;
3524 case 1:
3525 numchan = 2;
3526 break;
3527 case 2:
3528 numchan = 4;
3529 break;
3530 case 3:
3531 numchan = 8;
3532 break;
3533 }
3534 rdev->mc.vram_width = numchan * chansize;
3535 /* Could aper size report 0 ? */
01d73a69
JC		 3536 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3537 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3538 /* Setup GPU memory space */
05b3ef69
AD		 3539 if ((rdev->family == CHIP_PALM) ||
3540 (rdev->family == CHIP_SUMO) ||
3541 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3542 /* size in bytes on fusion */
3543 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3544 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3545 } else {
05b3ef69 3546 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3547 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3548 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3549 }
51e5fcd3 3550 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3551 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3552 radeon_update_bandwidth_info(rdev);
3553
bcc1c2a1
AD		 3554 return 0;
3555}
d594e46a 3556
187e3593 3557void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3558{
64c56e8c 3559 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3560 RREG32(GRBM_STATUS));
64c56e8c 3561 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3562 RREG32(GRBM_STATUS_SE0));
64c56e8c 3563 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3564 RREG32(GRBM_STATUS_SE1));
64c56e8c 3565 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3566 RREG32(SRBM_STATUS));
a65a4369
AD		 3567 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3568 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3569 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3570 RREG32(CP_STALLED_STAT1));
3571 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3572 RREG32(CP_STALLED_STAT2));
3573 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3574 RREG32(CP_BUSY_STAT));
3575 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3576 RREG32(CP_STAT));
eaaa6983
JG		 3577 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3578 RREG32(DMA_STATUS_REG));
168757ea
AD		 3579 if (rdev->family >= CHIP_CAYMAN) {
3580 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3581 RREG32(DMA_STATUS_REG + 0x800));
3582 }
0ecebb9e
AD		 3583}
3584
168757ea 3585bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3586{
a65a4369
AD		 3587 u32 crtc_hung = 0;
3588 u32 crtc_status[6];
3589 u32 i, j, tmp;
3590
3591 for (i = 0; i < rdev->num_crtc; i++) {
3592 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3593 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3594 crtc_hung |= (1 << i);
3595 }
3596 }
3597
3598 for (j = 0; j < 10; j++) {
3599 for (i = 0; i < rdev->num_crtc; i++) {
3600 if (crtc_hung & (1 << i)) {
3601 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3602 if (tmp != crtc_status[i])
3603 crtc_hung &= ~(1 << i);
3604 }
3605 }
3606 if (crtc_hung == 0)
3607 return false;
3608 udelay(100);
3609 }
3610
3611 return true;
3612}
3613
3614static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3615{
3616 u32 reset_mask = 0;
b7630473 3617 u32 tmp;
0ecebb9e 3618
a65a4369
AD		 3619 /* GRBM_STATUS */
3620 tmp = RREG32(GRBM_STATUS);
3621 if (tmp & (PA_BUSY | SC_BUSY |
3622 SH_BUSY | SX_BUSY |
3623 TA_BUSY | VGT_BUSY |
3624 DB_BUSY | CB_BUSY |
3625 SPI_BUSY | VGT_BUSY_NO_DMA))
3626 reset_mask |= RADEON_RESET_GFX;
3627
3628 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3629 CP_BUSY | CP_COHERENCY_BUSY))
3630 reset_mask |= RADEON_RESET_CP;
3631
3632 if (tmp & GRBM_EE_BUSY)
3633 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3634
a65a4369
AD		 3635 /* DMA_STATUS_REG */
3636 tmp = RREG32(DMA_STATUS_REG);
3637 if (!(tmp & DMA_IDLE))
3638 reset_mask |= RADEON_RESET_DMA;
3639
3640 /* SRBM_STATUS2 */
3641 tmp = RREG32(SRBM_STATUS2);
3642 if (tmp & DMA_BUSY)
3643 reset_mask |= RADEON_RESET_DMA;
3644
3645 /* SRBM_STATUS */
3646 tmp = RREG32(SRBM_STATUS);
3647 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3648 reset_mask |= RADEON_RESET_RLC;
3649
3650 if (tmp & IH_BUSY)
3651 reset_mask |= RADEON_RESET_IH;
3652
3653 if (tmp & SEM_BUSY)
3654 reset_mask |= RADEON_RESET_SEM;
3655
3656 if (tmp & GRBM_RQ_PENDING)
3657 reset_mask |= RADEON_RESET_GRBM;
3658
3659 if (tmp & VMC_BUSY)
3660 reset_mask |= RADEON_RESET_VMC;
3661
3662 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3663 MCC_BUSY | MCD_BUSY))
3664 reset_mask |= RADEON_RESET_MC;
3665
3666 if (evergreen_is_display_hung(rdev))
3667 reset_mask |= RADEON_RESET_DISPLAY;
3668
3669 /* VM_L2_STATUS */
3670 tmp = RREG32(VM_L2_STATUS);
3671 if (tmp & L2_BUSY)
3672 reset_mask |= RADEON_RESET_VMC;
3673
d808fc88
AD		 3674 /* Skip MC reset as it's mostly likely not hung, just busy */
3675 if (reset_mask & RADEON_RESET_MC) {
3676 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3677 reset_mask &= ~RADEON_RESET_MC;
3678 }
3679
a65a4369
AD		 3680 return reset_mask;
3681}
3682
3683static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3684{
3685 struct evergreen_mc_save save;
3686 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3687 u32 tmp;
19fc42ed 3688
0ecebb9e 3689 if (reset_mask == 0)
a65a4369 3690 return;
0ecebb9e
AD		 3691
3692 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3693
b7630473
AD		 3694 evergreen_print_gpu_status_regs(rdev);
3695
b7630473
AD		 3696 /* Disable CP parsing/prefetching */
3697 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3698
3699 if (reset_mask & RADEON_RESET_DMA) {
3700 /* Disable DMA */
3701 tmp = RREG32(DMA_RB_CNTL);
3702 tmp &= ~DMA_RB_ENABLE;
3703 WREG32(DMA_RB_CNTL, tmp);
3704 }
3705
b21b6e7a
AD		 3706 udelay(50);
3707
3708 evergreen_mc_stop(rdev, &save);
3709 if (evergreen_mc_wait_for_idle(rdev)) {
3710 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3711 }
3712
b7630473
AD		 3713 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3714 grbm_soft_reset |= SOFT_RESET_DB |
3715 SOFT_RESET_CB |
3716 SOFT_RESET_PA |
3717 SOFT_RESET_SC |
3718 SOFT_RESET_SPI |
3719 SOFT_RESET_SX |
3720 SOFT_RESET_SH |
3721 SOFT_RESET_TC |
3722 SOFT_RESET_TA |
3723 SOFT_RESET_VC |
3724 SOFT_RESET_VGT;
3725 }
3726
3727 if (reset_mask & RADEON_RESET_CP) {
3728 grbm_soft_reset |= SOFT_RESET_CP |
3729 SOFT_RESET_VGT;
3730
3731 srbm_soft_reset |= SOFT_RESET_GRBM;
3732 }
0ecebb9e
AD		 3733
3734 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3735 srbm_soft_reset |= SOFT_RESET_DMA;
3736
a65a4369
AD		 3737 if (reset_mask & RADEON_RESET_DISPLAY)
3738 srbm_soft_reset |= SOFT_RESET_DC;
3739
3740 if (reset_mask & RADEON_RESET_RLC)
3741 srbm_soft_reset |= SOFT_RESET_RLC;
3742
3743 if (reset_mask & RADEON_RESET_SEM)
3744 srbm_soft_reset |= SOFT_RESET_SEM;
3745
3746 if (reset_mask & RADEON_RESET_IH)
3747 srbm_soft_reset |= SOFT_RESET_IH;
3748
3749 if (reset_mask & RADEON_RESET_GRBM)
3750 srbm_soft_reset |= SOFT_RESET_GRBM;
3751
3752 if (reset_mask & RADEON_RESET_VMC)
3753 srbm_soft_reset |= SOFT_RESET_VMC;
3754
24178ec4
AD		 3755 if (!(rdev->flags & RADEON_IS_IGP)) {
3756 if (reset_mask & RADEON_RESET_MC)
3757 srbm_soft_reset |= SOFT_RESET_MC;
3758 }
a65a4369 3759
b7630473
AD		 3760 if (grbm_soft_reset) {
3761 tmp = RREG32(GRBM_SOFT_RESET);
3762 tmp |= grbm_soft_reset;
3763 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3764 WREG32(GRBM_SOFT_RESET, tmp);
3765 tmp = RREG32(GRBM_SOFT_RESET);
3766
3767 udelay(50);
3768
3769 tmp &= ~grbm_soft_reset;
3770 WREG32(GRBM_SOFT_RESET, tmp);
3771 tmp = RREG32(GRBM_SOFT_RESET);
3772 }
3773
3774 if (srbm_soft_reset) {
3775 tmp = RREG32(SRBM_SOFT_RESET);
3776 tmp |= srbm_soft_reset;
3777 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3778 WREG32(SRBM_SOFT_RESET, tmp);
3779 tmp = RREG32(SRBM_SOFT_RESET);
3780
3781 udelay(50);
3782
3783 tmp &= ~srbm_soft_reset;
3784 WREG32(SRBM_SOFT_RESET, tmp);
3785 tmp = RREG32(SRBM_SOFT_RESET);
3786 }
0ecebb9e
AD		 3787
3788 /* Wait a little for things to settle down */
3789 udelay(50);
3790
747943ea 3791 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3792 udelay(50);
3793
b7630473 3794 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3795}
3796
a2d07b74 3797int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3798{
a65a4369
AD		 3799 u32 reset_mask;
3800
3801 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3802
3803 if (reset_mask)
3804 r600_set_bios_scratch_engine_hung(rdev, true);
3805
3806 evergreen_gpu_soft_reset(rdev, reset_mask);
3807
3808 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3809
3810 if (!reset_mask)
3811 r600_set_bios_scratch_engine_hung(rdev, false);
3812
3813 return 0;
747943ea
AD		 3814}
3815
123bc183
AD		 3816/**
3817 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3818 *
3819 * @rdev: radeon_device pointer
3820 * @ring: radeon_ring structure holding ring information
3821 *
3822 * Check if the GFX engine is locked up.
3823 * Returns true if the engine appears to be locked up, false if not.
3824 */
3825bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3826{
3827 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3828
3829 if (!(reset_mask & (RADEON_RESET_GFX |
3830 RADEON_RESET_COMPUTE |
3831 RADEON_RESET_CP))) {
3832 radeon_ring_lockup_update(ring);
3833 return false;
3834 }
3835 /* force CP activities */
3836 radeon_ring_force_activity(rdev, ring);
3837 return radeon_ring_test_lockup(rdev, ring);
3838}
3839
3840/**
3841 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3842 *
3843 * @rdev: radeon_device pointer
3844 * @ring: radeon_ring structure holding ring information
3845 *
3846 * Check if the async DMA engine is locked up.
3847 * Returns true if the engine appears to be locked up, false if not.
3848 */
3849bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3850{
3851 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3852
3853 if (!(reset_mask & RADEON_RESET_DMA)) {
3854 radeon_ring_lockup_update(ring);
3855 return false;
3856 }
3857 /* force ring activities */
3858 radeon_ring_force_activity(rdev, ring);
3859 return radeon_ring_test_lockup(rdev, ring);
3860}
3861
2948f5e6
AD		 3862/*
3863 * RLC
3864 */
3865#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3866#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3867
3868void sumo_rlc_fini(struct radeon_device *rdev)
3869{
3870 int r;
3871
3872 /* save restore block */
3873 if (rdev->rlc.save_restore_obj) {
3874 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3875 if (unlikely(r != 0))
3876 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3877 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3878 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3879
3880 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3881 rdev->rlc.save_restore_obj = NULL;
3882 }
3883
3884 /* clear state block */
3885 if (rdev->rlc.clear_state_obj) {
3886 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3887 if (unlikely(r != 0))
3888 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3889 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3890 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3891
3892 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3893 rdev->rlc.clear_state_obj = NULL;
3894 }
3895}
3896
3897int sumo_rlc_init(struct radeon_device *rdev)
3898{
3899 u32 *src_ptr;
3900 volatile u32 *dst_ptr;
3901 u32 dws, data, i, j, k, reg_num;
3902 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
3903 u64 reg_list_mc_addr;
3904 struct cs_section_def *cs_data;
3905 int r;
3906
3907 src_ptr = rdev->rlc.reg_list;
3908 dws = rdev->rlc.reg_list_size;
3909 cs_data = rdev->rlc.cs_data;
3910
3911 /* save restore block */
3912 if (rdev->rlc.save_restore_obj == NULL) {
3913 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3914 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3915 if (r) {
3916 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3917 return r;
3918 }
3919 }
3920
3921 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3922 if (unlikely(r != 0)) {
3923 sumo_rlc_fini(rdev);
3924 return r;
3925 }
3926 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3927 &rdev->rlc.save_restore_gpu_addr);
3928 if (r) {
3929 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3930 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3931 sumo_rlc_fini(rdev);
3932 return r;
3933 }
3934 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3935 if (r) {
3936 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3937 sumo_rlc_fini(rdev);
3938 return r;
3939 }
3940 /* write the sr buffer */
3941 dst_ptr = rdev->rlc.sr_ptr;
3942 /* format:
3943 * dw0: (reg2 << 16) | reg1
3944 * dw1: reg1 save space
3945 * dw2: reg2 save space
3946 */
3947 for (i = 0; i < dws; i++) {
3948 data = src_ptr[i] >> 2;
3949 i++;
3950 if (i < dws)
3951 data |= (src_ptr[i] >> 2) << 16;
3952 j = (((i - 1) * 3) / 2);
3953 dst_ptr[j] = data;
3954 }
3955 j = ((i * 3) / 2);
3956 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
3957
3958 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3959 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3960
3961 /* clear state block */
3962 reg_list_num = 0;
3963 dws = 0;
3964 for (i = 0; cs_data[i].section != NULL; i++) {
3965 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3966 reg_list_num++;
3967 dws += cs_data[i].section[j].reg_count;
3968 }
3969 }
3970 reg_list_blk_index = (3 * reg_list_num + 2);
3971 dws += reg_list_blk_index;
3972
3973 if (rdev->rlc.clear_state_obj == NULL) {
3974 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3975 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
3976 if (r) {
3977 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
3978 sumo_rlc_fini(rdev);
3979 return r;
3980 }
3981 }
3982 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3983 if (unlikely(r != 0)) {
3984 sumo_rlc_fini(rdev);
3985 return r;
3986 }
3987 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
3988 &rdev->rlc.clear_state_gpu_addr);
3989 if (r) {
3990
3991 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3992 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
3993 sumo_rlc_fini(rdev);
3994 return r;
3995 }
3996 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
3997 if (r) {
3998 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
3999 sumo_rlc_fini(rdev);
4000 return r;
4001 }
4002 /* set up the cs buffer */
4003 dst_ptr = rdev->rlc.cs_ptr;
4004 reg_list_hdr_blk_index = 0;
4005 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4006 data = upper_32_bits(reg_list_mc_addr);
4007 dst_ptr[reg_list_hdr_blk_index] = data;
4008 reg_list_hdr_blk_index++;
4009 for (i = 0; cs_data[i].section != NULL; i++) {
4010 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4011 reg_num = cs_data[i].section[j].reg_count;
4012 data = reg_list_mc_addr & 0xffffffff;
4013 dst_ptr[reg_list_hdr_blk_index] = data;
4014 reg_list_hdr_blk_index++;
4015
4016 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4017 dst_ptr[reg_list_hdr_blk_index] = data;
4018 reg_list_hdr_blk_index++;
4019
4020 data = 0x08000000 | (reg_num * 4);
4021 dst_ptr[reg_list_hdr_blk_index] = data;
4022 reg_list_hdr_blk_index++;
4023
4024 for (k = 0; k < reg_num; k++) {
4025 data = cs_data[i].section[j].extent[k];
4026 dst_ptr[reg_list_blk_index + k] = data;
4027 }
4028 reg_list_mc_addr += reg_num * 4;
4029 reg_list_blk_index += reg_num;
4030 }
4031 }
4032 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4033
4034 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4035 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4036
4037 return 0;
4038}
4039
4040static void evergreen_rlc_start(struct radeon_device *rdev)
4041{
8ba10463
AD		 4042 u32 mask = RLC_ENABLE;
4043
4044 if (rdev->flags & RADEON_IS_IGP) {
4045 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4046 if (rdev->family == CHIP_ARUBA)
4047 mask |= DYN_PER_SIMD_PG_ENABLE | LB_CNT_SPIM_ACTIVE | LOAD_BALANCE_ENABLE;
4048 }
4049
4050 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4051}
4052
4053int evergreen_rlc_resume(struct radeon_device *rdev)
4054{
4055 u32 i;
4056 const __be32 *fw_data;
4057
4058 if (!rdev->rlc_fw)
4059 return -EINVAL;
4060
4061 r600_rlc_stop(rdev);
4062
4063 WREG32(RLC_HB_CNTL, 0);
4064
4065 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4066 if (rdev->family == CHIP_ARUBA) {
4067 u32 always_on_bitmap =
4068 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4069 /* find out the number of active simds */
4070 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4071 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4072 tmp = hweight32(~tmp);
4073 if (tmp == rdev->config.cayman.max_simds_per_se) {
4074 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4075 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4076 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4077 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4078 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4079 }
4080 } else {
4081 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4082 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4083 }
2948f5e6
AD		 4084 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4085 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4086 } else {
4087 WREG32(RLC_HB_BASE, 0);
4088 WREG32(RLC_HB_RPTR, 0);
4089 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4090 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4091 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4092 }
2948f5e6
AD		 4093 WREG32(RLC_MC_CNTL, 0);
4094 WREG32(RLC_UCODE_CNTL, 0);
4095
4096 fw_data = (const __be32 *)rdev->rlc_fw->data;
4097 if (rdev->family >= CHIP_ARUBA) {
4098 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4099 WREG32(RLC_UCODE_ADDR, i);
4100 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4101 }
4102 } else if (rdev->family >= CHIP_CAYMAN) {
4103 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4104 WREG32(RLC_UCODE_ADDR, i);
4105 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4106 }
4107 } else {
4108 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4109 WREG32(RLC_UCODE_ADDR, i);
4110 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4111 }
4112 }
4113 WREG32(RLC_UCODE_ADDR, 0);
4114
4115 evergreen_rlc_start(rdev);
4116
4117 return 0;
4118}
4119
45f9a39b
AD		 4120/* Interrupts */
4121
4122u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4123{
46437057 4124 if (crtc >= rdev->num_crtc)
45f9a39b 4125 return 0;
46437057
AD		 4126 else
4127 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4128}
4129
4130void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4131{
4132 u32 tmp;
4133
1b37078b
AD		 4134 if (rdev->family >= CHIP_CAYMAN) {
4135 cayman_cp_int_cntl_setup(rdev, 0,
4136 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4137 cayman_cp_int_cntl_setup(rdev, 1, 0);
4138 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4139 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4140 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4141 } else
4142 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4143 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4144 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 4145 WREG32(GRBM_INT_CNTL, 0);
4146 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4147 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4148 if (rdev->num_crtc >= 4) {
18007401
AD		 4149 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4150 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4151 }
4152 if (rdev->num_crtc >= 6) {
18007401
AD		 4153 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4154 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4155 }
45f9a39b
AD		 4156
4157 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4158 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4159 if (rdev->num_crtc >= 4) {
18007401
AD		 4160 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4161 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4162 }
4163 if (rdev->num_crtc >= 6) {
18007401
AD		 4164 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4165 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4166 }
45f9a39b 4167
05b3ef69
AD		 4168 /* only one DAC on DCE6 */
4169 if (!ASIC_IS_DCE6(rdev))
4170 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4171 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4172
4173 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4174 WREG32(DC_HPD1_INT_CONTROL, tmp);
4175 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4176 WREG32(DC_HPD2_INT_CONTROL, tmp);
4177 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4178 WREG32(DC_HPD3_INT_CONTROL, tmp);
4179 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4180 WREG32(DC_HPD4_INT_CONTROL, tmp);
4181 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4182 WREG32(DC_HPD5_INT_CONTROL, tmp);
4183 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4184 WREG32(DC_HPD6_INT_CONTROL, tmp);
4185
4186}
4187
4188int evergreen_irq_set(struct radeon_device *rdev)
4189{
4190 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4191 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4192 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4193 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4194 u32 grbm_int_cntl = 0;
6f34be50 4195 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
f122c610 4196 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4197 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4198 u32 thermal_int = 0;
45f9a39b
AD		 4199
4200 if (!rdev->irq.installed) {
fce7d61b 4201 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4202 return -EINVAL;
4203 }
4204 /* don't enable anything if the ih is disabled */
4205 if (!rdev->ih.enabled) {
4206 r600_disable_interrupts(rdev);
4207 /* force the active interrupt state to all disabled */
4208 evergreen_disable_interrupt_state(rdev);
4209 return 0;
4210 }
4211
4212 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4213 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4214 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4215 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4216 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4217 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
d70229f7
AD		 4218 if (rdev->family == CHIP_ARUBA)
4219 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4220 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4221 else
4222 thermal_int = RREG32(CG_THERMAL_INT) &
4223 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4224
f122c610
AD		 4225 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4226 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4227 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4228 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4229 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4230 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4231
233d1ad5
AD		 4232 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4233
1b37078b
AD		 4234 if (rdev->family >= CHIP_CAYMAN) {
4235 /* enable CP interrupts on all rings */
736fc37f 4236 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4237 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4238 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4239 }
736fc37f 4240 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4241 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4242 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4243 }
736fc37f 4244 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4245 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4246 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4247 }
4248 } else {
736fc37f 4249 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4250 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4251 cp_int_cntl |= RB_INT_ENABLE;
4252 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4253 }
45f9a39b 4254 }
1b37078b 4255
233d1ad5
AD		 4256 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4257 DRM_DEBUG("r600_irq_set: sw int dma\n");
4258 dma_cntl |= TRAP_ENABLE;
4259 }
4260
f60cbd11
AD		 4261 if (rdev->family >= CHIP_CAYMAN) {
4262 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4263 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4264 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4265 dma_cntl1 |= TRAP_ENABLE;
4266 }
4267 }
4268
dc50ba7f
AD		 4269 if (rdev->irq.dpm_thermal) {
4270 DRM_DEBUG("dpm thermal\n");
4271 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4272 }
4273
6f34be50 4274 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4275 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4276 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4277 crtc1 |= VBLANK_INT_MASK;
4278 }
6f34be50 4279 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4280 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4281 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4282 crtc2 |= VBLANK_INT_MASK;
4283 }
6f34be50 4284 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4285 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4286 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4287 crtc3 |= VBLANK_INT_MASK;
4288 }
6f34be50 4289 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4290 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4291 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4292 crtc4 |= VBLANK_INT_MASK;
4293 }
6f34be50 4294 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4295 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4296 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4297 crtc5 |= VBLANK_INT_MASK;
4298 }
6f34be50 4299 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4300 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4301 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4302 crtc6 |= VBLANK_INT_MASK;
4303 }
4304 if (rdev->irq.hpd[0]) {
4305 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4306 hpd1 |= DC_HPDx_INT_EN;
4307 }
4308 if (rdev->irq.hpd[1]) {
4309 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4310 hpd2 |= DC_HPDx_INT_EN;
4311 }
4312 if (rdev->irq.hpd[2]) {
4313 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4314 hpd3 |= DC_HPDx_INT_EN;
4315 }
4316 if (rdev->irq.hpd[3]) {
4317 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4318 hpd4 |= DC_HPDx_INT_EN;
4319 }
4320 if (rdev->irq.hpd[4]) {
4321 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4322 hpd5 |= DC_HPDx_INT_EN;
4323 }
4324 if (rdev->irq.hpd[5]) {
4325 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4326 hpd6 |= DC_HPDx_INT_EN;
4327 }
f122c610
AD		 4328 if (rdev->irq.afmt[0]) {
4329 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4330 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4331 }
4332 if (rdev->irq.afmt[1]) {
4333 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4334 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4335 }
4336 if (rdev->irq.afmt[2]) {
4337 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4338 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4339 }
4340 if (rdev->irq.afmt[3]) {
4341 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4342 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4343 }
4344 if (rdev->irq.afmt[4]) {
4345 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4346 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4347 }
4348 if (rdev->irq.afmt[5]) {
4349 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4350 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4351 }
45f9a39b 4352
1b37078b
AD		 4353 if (rdev->family >= CHIP_CAYMAN) {
4354 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4355 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4356 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4357 } else
4358 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4359
4360 WREG32(DMA_CNTL, dma_cntl);
4361
f60cbd11
AD		 4362 if (rdev->family >= CHIP_CAYMAN)
4363 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4364
2031f77c 4365 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4366
4367 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4368 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4369 if (rdev->num_crtc >= 4) {
18007401
AD		 4370 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4371 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4372 }
4373 if (rdev->num_crtc >= 6) {
18007401
AD		 4374 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4375 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4376 }
45f9a39b 4377
6f34be50
AD		 4378 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4379 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
b7eff394
AD		 4380 if (rdev->num_crtc >= 4) {
4381 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4382 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4383 }
4384 if (rdev->num_crtc >= 6) {
4385 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4386 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4387 }
6f34be50 4388
45f9a39b
AD		 4389 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4390 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4391 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4392 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4393 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4394 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4395 if (rdev->family == CHIP_ARUBA)
4396 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4397 else
4398 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4399
f122c610
AD		 4400 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4401 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4402 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4403 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4404 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4405 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4406
bcc1c2a1
AD		 4407 return 0;
4408}
4409
cbdd4501 4410static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4411{
4412 u32 tmp;
4413
6f34be50
AD		 4414 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4415 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4416 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4417 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4418 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4419 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4420 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4421 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4422 if (rdev->num_crtc >= 4) {
4423 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4424 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4425 }
4426 if (rdev->num_crtc >= 6) {
4427 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4428 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4429 }
6f34be50 4430
f122c610
AD		 4431 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4432 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4433 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4434 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4435 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4436 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4437
6f34be50
AD		 4438 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4439 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4440 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4441 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4442 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4443 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4444 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4445 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4446 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4447 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4448 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4449 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4450
b7eff394
AD		 4451 if (rdev->num_crtc >= 4) {
4452 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4453 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4454 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4455 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4456 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4457 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4458 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4459 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4460 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4461 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4462 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4463 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4464 }
4465
4466 if (rdev->num_crtc >= 6) {
4467 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4468 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4469 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4470 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4471 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4472 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4473 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4474 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4475 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4476 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4477 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4478 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4479 }
45f9a39b 4480
6f34be50 4481 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4482 tmp = RREG32(DC_HPD1_INT_CONTROL);
4483 tmp |= DC_HPDx_INT_ACK;
4484 WREG32(DC_HPD1_INT_CONTROL, tmp);
4485 }
6f34be50 4486 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4487 tmp = RREG32(DC_HPD2_INT_CONTROL);
4488 tmp |= DC_HPDx_INT_ACK;
4489 WREG32(DC_HPD2_INT_CONTROL, tmp);
4490 }
6f34be50 4491 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4492 tmp = RREG32(DC_HPD3_INT_CONTROL);
4493 tmp |= DC_HPDx_INT_ACK;
4494 WREG32(DC_HPD3_INT_CONTROL, tmp);
4495 }
6f34be50 4496 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4497 tmp = RREG32(DC_HPD4_INT_CONTROL);
4498 tmp |= DC_HPDx_INT_ACK;
4499 WREG32(DC_HPD4_INT_CONTROL, tmp);
4500 }
6f34be50 4501 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4502 tmp = RREG32(DC_HPD5_INT_CONTROL);
4503 tmp |= DC_HPDx_INT_ACK;
4504 WREG32(DC_HPD5_INT_CONTROL, tmp);
4505 }
6f34be50 4506 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4507 tmp = RREG32(DC_HPD5_INT_CONTROL);
4508 tmp |= DC_HPDx_INT_ACK;
4509 WREG32(DC_HPD6_INT_CONTROL, tmp);
4510 }
f122c610
AD		 4511 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4512 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4513 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4514 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4515 }
4516 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4517 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4518 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4519 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4520 }
4521 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4522 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4523 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4524 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4525 }
4526 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4527 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4528 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4529 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4530 }
4531 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4532 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4533 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4534 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4535 }
4536 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4537 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4538 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4539 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4540 }
45f9a39b
AD		 4541}
4542
1109ca09 4543static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4544{
45f9a39b
AD		 4545 r600_disable_interrupts(rdev);
4546 /* Wait and acknowledge irq */
4547 mdelay(1);
6f34be50 4548 evergreen_irq_ack(rdev);
45f9a39b
AD		 4549 evergreen_disable_interrupt_state(rdev);
4550}
4551
755d819e 4552void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4553{
4554 evergreen_irq_disable(rdev);
4555 r600_rlc_stop(rdev);
4556}
4557
cbdd4501 4558static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4559{
4560 u32 wptr, tmp;
4561
724c80e1 4562 if (rdev->wb.enabled)
204ae24d 4563 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4564 else
4565 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4566
4567 if (wptr & RB_OVERFLOW) {
4568 /* When a ring buffer overflow happen start parsing interrupt
4569 * from the last not overwritten vector (wptr + 16). Hopefully
4570 * this should allow us to catchup.
4571 */
4572 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4573 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4574 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4575 tmp = RREG32(IH_RB_CNTL);
4576 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4577 WREG32(IH_RB_CNTL, tmp);
4578 }
4579 return (wptr & rdev->ih.ptr_mask);
4580}
4581
4582int evergreen_irq_process(struct radeon_device *rdev)
4583{
682f1a54
DA		 4584 u32 wptr;
4585 u32 rptr;
45f9a39b
AD		 4586 u32 src_id, src_data;
4587 u32 ring_index;
45f9a39b 4588 bool queue_hotplug = false;
f122c610 4589 bool queue_hdmi = false;
dc50ba7f 4590 bool queue_thermal = false;
45f9a39b 4591
682f1a54 4592 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4593 return IRQ_NONE;
4594
682f1a54 4595 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4596
4597restart_ih:
4598 /* is somebody else already processing irqs? */
4599 if (atomic_xchg(&rdev->ih.lock, 1))
4600 return IRQ_NONE;
4601
682f1a54
DA		 4602 rptr = rdev->ih.rptr;
4603 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4604
964f6645
BH		 4605 /* Order reading of wptr vs. reading of IH ring data */
4606 rmb();
4607
45f9a39b 4608 /* display interrupts */
6f34be50 4609 evergreen_irq_ack(rdev);
45f9a39b 4610
45f9a39b
AD		 4611 while (rptr != wptr) {
4612 /* wptr/rptr are in bytes! */
4613 ring_index = rptr / 4;
0f234f5f
AD		 4614 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4615 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4616
4617 switch (src_id) {
4618 case 1: /* D1 vblank/vline */
4619 switch (src_data) {
4620 case 0: /* D1 vblank */
6f34be50 4621 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4622 if (rdev->irq.crtc_vblank_int[0]) {
4623 drm_handle_vblank(rdev->ddev, 0);
4624 rdev->pm.vblank_sync = true;
4625 wake_up(&rdev->irq.vblank_queue);
4626 }
736fc37f 4627 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4628 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4629 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4630 DRM_DEBUG("IH: D1 vblank\n");
4631 }
4632 break;
4633 case 1: /* D1 vline */
6f34be50
AD		 4634 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4635 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4636 DRM_DEBUG("IH: D1 vline\n");
4637 }
4638 break;
4639 default:
4640 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4641 break;
4642 }
4643 break;
4644 case 2: /* D2 vblank/vline */
4645 switch (src_data) {
4646 case 0: /* D2 vblank */
6f34be50 4647 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4648 if (rdev->irq.crtc_vblank_int[1]) {
4649 drm_handle_vblank(rdev->ddev, 1);
4650 rdev->pm.vblank_sync = true;
4651 wake_up(&rdev->irq.vblank_queue);
4652 }
736fc37f 4653 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4654 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4655 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4656 DRM_DEBUG("IH: D2 vblank\n");
4657 }
4658 break;
4659 case 1: /* D2 vline */
6f34be50
AD		 4660 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4661 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4662 DRM_DEBUG("IH: D2 vline\n");
4663 }
4664 break;
4665 default:
4666 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4667 break;
4668 }
4669 break;
4670 case 3: /* D3 vblank/vline */
4671 switch (src_data) {
4672 case 0: /* D3 vblank */
6f34be50
AD		 4673 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4674 if (rdev->irq.crtc_vblank_int[2]) {
4675 drm_handle_vblank(rdev->ddev, 2);
4676 rdev->pm.vblank_sync = true;
4677 wake_up(&rdev->irq.vblank_queue);
4678 }
736fc37f 4679 if (atomic_read(&rdev->irq.pflip[2]))
6f34be50
AD		 4680 radeon_crtc_handle_flip(rdev, 2);
4681 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4682 DRM_DEBUG("IH: D3 vblank\n");
4683 }
4684 break;
4685 case 1: /* D3 vline */
6f34be50
AD		 4686 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4687 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4688 DRM_DEBUG("IH: D3 vline\n");
4689 }
4690 break;
4691 default:
4692 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4693 break;
4694 }
4695 break;
4696 case 4: /* D4 vblank/vline */
4697 switch (src_data) {
4698 case 0: /* D4 vblank */
6f34be50
AD		 4699 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4700 if (rdev->irq.crtc_vblank_int[3]) {
4701 drm_handle_vblank(rdev->ddev, 3);
4702 rdev->pm.vblank_sync = true;
4703 wake_up(&rdev->irq.vblank_queue);
4704 }
736fc37f 4705 if (atomic_read(&rdev->irq.pflip[3]))
6f34be50
AD		 4706 radeon_crtc_handle_flip(rdev, 3);
4707 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4708 DRM_DEBUG("IH: D4 vblank\n");
4709 }
4710 break;
4711 case 1: /* D4 vline */
6f34be50
AD		 4712 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4713 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4714 DRM_DEBUG("IH: D4 vline\n");
4715 }
4716 break;
4717 default:
4718 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4719 break;
4720 }
4721 break;
4722 case 5: /* D5 vblank/vline */
4723 switch (src_data) {
4724 case 0: /* D5 vblank */
6f34be50
AD		 4725 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4726 if (rdev->irq.crtc_vblank_int[4]) {
4727 drm_handle_vblank(rdev->ddev, 4);
4728 rdev->pm.vblank_sync = true;
4729 wake_up(&rdev->irq.vblank_queue);
4730 }
736fc37f 4731 if (atomic_read(&rdev->irq.pflip[4]))
6f34be50
AD		 4732 radeon_crtc_handle_flip(rdev, 4);
4733 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4734 DRM_DEBUG("IH: D5 vblank\n");
4735 }
4736 break;
4737 case 1: /* D5 vline */
6f34be50
AD		 4738 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4739 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4740 DRM_DEBUG("IH: D5 vline\n");
4741 }
4742 break;
4743 default:
4744 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4745 break;
4746 }
4747 break;
4748 case 6: /* D6 vblank/vline */
4749 switch (src_data) {
4750 case 0: /* D6 vblank */
6f34be50
AD		 4751 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4752 if (rdev->irq.crtc_vblank_int[5]) {
4753 drm_handle_vblank(rdev->ddev, 5);
4754 rdev->pm.vblank_sync = true;
4755 wake_up(&rdev->irq.vblank_queue);
4756 }
736fc37f 4757 if (atomic_read(&rdev->irq.pflip[5]))
6f34be50
AD		 4758 radeon_crtc_handle_flip(rdev, 5);
4759 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4760 DRM_DEBUG("IH: D6 vblank\n");
4761 }
4762 break;
4763 case 1: /* D6 vline */
6f34be50
AD		 4764 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4765 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4766 DRM_DEBUG("IH: D6 vline\n");
4767 }
4768 break;
4769 default:
4770 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4771 break;
4772 }
4773 break;
4774 case 42: /* HPD hotplug */
4775 switch (src_data) {
4776 case 0:
6f34be50
AD		 4777 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4778 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4779 queue_hotplug = true;
4780 DRM_DEBUG("IH: HPD1\n");
4781 }
4782 break;
4783 case 1:
6f34be50
AD		 4784 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4785 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4786 queue_hotplug = true;
4787 DRM_DEBUG("IH: HPD2\n");
4788 }
4789 break;
4790 case 2:
6f34be50
AD		 4791 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4792 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4793 queue_hotplug = true;
4794 DRM_DEBUG("IH: HPD3\n");
4795 }
4796 break;
4797 case 3:
6f34be50
AD		 4798 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4799 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 4800 queue_hotplug = true;
4801 DRM_DEBUG("IH: HPD4\n");
4802 }
4803 break;
4804 case 4:
6f34be50
AD		 4805 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4806 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 4807 queue_hotplug = true;
4808 DRM_DEBUG("IH: HPD5\n");
4809 }
4810 break;
4811 case 5:
6f34be50
AD		 4812 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4813 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 4814 queue_hotplug = true;
4815 DRM_DEBUG("IH: HPD6\n");
4816 }
4817 break;
4818 default:
4819 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4820 break;
4821 }
4822 break;
f122c610
AD		 4823 case 44: /* hdmi */
4824 switch (src_data) {
4825 case 0:
4826 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4827 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4828 queue_hdmi = true;
4829 DRM_DEBUG("IH: HDMI0\n");
4830 }
4831 break;
4832 case 1:
4833 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4834 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4835 queue_hdmi = true;
4836 DRM_DEBUG("IH: HDMI1\n");
4837 }
4838 break;
4839 case 2:
4840 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4841 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4842 queue_hdmi = true;
4843 DRM_DEBUG("IH: HDMI2\n");
4844 }
4845 break;
4846 case 3:
4847 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4848 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4849 queue_hdmi = true;
4850 DRM_DEBUG("IH: HDMI3\n");
4851 }
4852 break;
4853 case 4:
4854 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4855 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4856 queue_hdmi = true;
4857 DRM_DEBUG("IH: HDMI4\n");
4858 }
4859 break;
4860 case 5:
4861 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4862 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4863 queue_hdmi = true;
4864 DRM_DEBUG("IH: HDMI5\n");
4865 }
4866 break;
4867 default:
4868 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4869 break;
4870 }
f2ba57b5
CK		 4871 case 124: /* UVD */
4872 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4873 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 4874 break;
ae133a11
CK		 4875 case 146:
4876 case 147:
4877 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4878 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4879 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4880 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4881 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4882 /* reset addr and status */
4883 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4884 break;
45f9a39b
AD		 4885 case 176: /* CP_INT in ring buffer */
4886 case 177: /* CP_INT in IB1 */
4887 case 178: /* CP_INT in IB2 */
4888 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 4889 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 4890 break;
4891 case 181: /* CP EOP event */
4892 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 4893 if (rdev->family >= CHIP_CAYMAN) {
4894 switch (src_data) {
4895 case 0:
4896 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4897 break;
4898 case 1:
4899 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4900 break;
4901 case 2:
4902 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4903 break;
4904 }
4905 } else
4906 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 4907 break;
233d1ad5
AD		 4908 case 224: /* DMA trap event */
4909 DRM_DEBUG("IH: DMA trap\n");
4910 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4911 break;
dc50ba7f
AD		 4912 case 230: /* thermal low to high */
4913 DRM_DEBUG("IH: thermal low to high\n");
4914 rdev->pm.dpm.thermal.high_to_low = false;
4915 queue_thermal = true;
4916 break;
4917 case 231: /* thermal high to low */
4918 DRM_DEBUG("IH: thermal high to low\n");
4919 rdev->pm.dpm.thermal.high_to_low = true;
4920 queue_thermal = true;
4921 break;
2031f77c 4922 case 233: /* GUI IDLE */
303c805c 4923 DRM_DEBUG("IH: GUI idle\n");
2031f77c 4924 break;
f60cbd11
AD		 4925 case 244: /* DMA trap event */
4926 if (rdev->family >= CHIP_CAYMAN) {
4927 DRM_DEBUG("IH: DMA1 trap\n");
4928 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4929 }
4930 break;
45f9a39b
AD		 4931 default:
4932 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4933 break;
4934 }
4935
4936 /* wptr/rptr are in bytes! */
4937 rptr += 16;
4938 rptr &= rdev->ih.ptr_mask;
4939 }
45f9a39b 4940 if (queue_hotplug)
32c87fca 4941 schedule_work(&rdev->hotplug_work);
f122c610
AD		 4942 if (queue_hdmi)
4943 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 4944 if (queue_thermal && rdev->pm.dpm_enabled)
4945 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b
AD		 4946 rdev->ih.rptr = rptr;
4947 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 4948 atomic_set(&rdev->ih.lock, 0);
4949
4950 /* make sure wptr hasn't changed while processing */
4951 wptr = evergreen_get_ih_wptr(rdev);
4952 if (wptr != rptr)
4953 goto restart_ih;
4954
45f9a39b
AD		 4955 return IRQ_HANDLED;
4956}
4957
233d1ad5
AD		 4958/**
4959 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
4960 *
4961 * @rdev: radeon_device pointer
4962 * @fence: radeon fence object
4963 *
4964 * Add a DMA fence packet to the ring to write
4965 * the fence seq number and DMA trap packet to generate
4966 * an interrupt if needed (evergreen-SI).
4967 */
4968void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
4969 struct radeon_fence *fence)
4970{
4971 struct radeon_ring *ring = &rdev->ring[fence->ring];
4972 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4973 /* write the fence */
0fcb6155 4974 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
233d1ad5
AD		 4975 radeon_ring_write(ring, addr & 0xfffffffc);
4976 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
4977 radeon_ring_write(ring, fence->seq);
4978 /* generate an interrupt */
0fcb6155 4979 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
233d1ad5 4980 /* flush HDP */
0fcb6155 4981 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4b681c28 4982 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
233d1ad5
AD		 4983 radeon_ring_write(ring, 1);
4984}
4985
4986/**
4987 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
4988 *
4989 * @rdev: radeon_device pointer
4990 * @ib: IB object to schedule
4991 *
4992 * Schedule an IB in the DMA ring (evergreen).
4993 */
4994void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
4995 struct radeon_ib *ib)
4996{
4997 struct radeon_ring *ring = &rdev->ring[ib->ring];
4998
4999 if (rdev->wb.enabled) {
5000 u32 next_rptr = ring->wptr + 4;
5001 while ((next_rptr & 7) != 5)
5002 next_rptr++;
5003 next_rptr += 3;
0fcb6155 5004 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
233d1ad5
AD		 5005 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5006 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
5007 radeon_ring_write(ring, next_rptr);
5008 }
5009
5010 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
5011 * Pad as necessary with NOPs.
5012 */
5013 while ((ring->wptr & 7) != 5)
0fcb6155
JG		 5014 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5015 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
233d1ad5
AD		 5016 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
5017 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
5018
5019}
5020
5021/**
5022 * evergreen_copy_dma - copy pages using the DMA engine
5023 *
5024 * @rdev: radeon_device pointer
5025 * @src_offset: src GPU address
5026 * @dst_offset: dst GPU address
5027 * @num_gpu_pages: number of GPU pages to xfer
5028 * @fence: radeon fence object
5029 *
5030 * Copy GPU paging using the DMA engine (evergreen-cayman).
5031 * Used by the radeon ttm implementation to move pages if
5032 * registered as the asic copy callback.
5033 */
5034int evergreen_copy_dma(struct radeon_device *rdev,
5035 uint64_t src_offset, uint64_t dst_offset,
5036 unsigned num_gpu_pages,
5037 struct radeon_fence **fence)
5038{
5039 struct radeon_semaphore *sem = NULL;
5040 int ring_index = rdev->asic->copy.dma_ring_index;
5041 struct radeon_ring *ring = &rdev->ring[ring_index];
5042 u32 size_in_dw, cur_size_in_dw;
5043 int i, num_loops;
5044 int r = 0;
5045
5046 r = radeon_semaphore_create(rdev, &sem);
5047 if (r) {
5048 DRM_ERROR("radeon: moving bo (%d).\n", r);
5049 return r;
5050 }
5051
5052 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
5053 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
5054 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5055 if (r) {
5056 DRM_ERROR("radeon: moving bo (%d).\n", r);
5057 radeon_semaphore_free(rdev, &sem, NULL);
5058 return r;
5059 }
5060
5061 if (radeon_fence_need_sync(*fence, ring->idx)) {
5062 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5063 ring->idx);
5064 radeon_fence_note_sync(*fence, ring->idx);
5065 } else {
5066 radeon_semaphore_free(rdev, &sem, NULL);
5067 }
5068
5069 for (i = 0; i < num_loops; i++) {
5070 cur_size_in_dw = size_in_dw;
5071 if (cur_size_in_dw > 0xFFFFF)
5072 cur_size_in_dw = 0xFFFFF;
5073 size_in_dw -= cur_size_in_dw;
0fcb6155 5074 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
233d1ad5
AD		 5075 radeon_ring_write(ring, dst_offset & 0xfffffffc);
5076 radeon_ring_write(ring, src_offset & 0xfffffffc);
5077 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5078 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5079 src_offset += cur_size_in_dw * 4;
5080 dst_offset += cur_size_in_dw * 4;
5081 }
5082
5083 r = radeon_fence_emit(rdev, fence, ring->idx);
5084 if (r) {
5085 radeon_ring_unlock_undo(rdev, ring);
5086 return r;
5087 }
5088
5089 radeon_ring_unlock_commit(rdev, ring);
5090 radeon_semaphore_free(rdev, &sem, *fence);
5091
5092 return r;
5093}
5094
bcc1c2a1
AD		 5095static int evergreen_startup(struct radeon_device *rdev)
5096{
f2ba57b5 5097 struct radeon_ring *ring;
bcc1c2a1
AD		 5098 int r;
5099
9e46a48d 5100 /* enable pcie gen2 link */
cd54033a 5101 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5102 /* enable aspm */
5103 evergreen_program_aspm(rdev);
9e46a48d 5104
0af62b01
AD		 5105 if (ASIC_IS_DCE5(rdev)) {
5106 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5107 r = ni_init_microcode(rdev);
5108 if (r) {
5109 DRM_ERROR("Failed to load firmware!\n");
5110 return r;
5111 }
5112 }
755d819e 5113 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5114 if (r) {
0af62b01 5115 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5116 return r;
5117 }
0af62b01
AD		 5118 } else {
5119 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5120 r = r600_init_microcode(rdev);
5121 if (r) {
5122 DRM_ERROR("Failed to load firmware!\n");
5123 return r;
5124 }
5125 }
bcc1c2a1 5126 }
fe251e2f 5127
16cdf04d
AD		 5128 r = r600_vram_scratch_init(rdev);
5129 if (r)
5130 return r;
5131
bcc1c2a1 5132 evergreen_mc_program(rdev);
bcc1c2a1 5133 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5134 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5135 } else {
5136 r = evergreen_pcie_gart_enable(rdev);
5137 if (r)
5138 return r;
5139 }
bcc1c2a1 5140 evergreen_gpu_init(rdev);
bcc1c2a1 5141
d7ccd8fc 5142 r = evergreen_blit_init(rdev);
bcc1c2a1 5143 if (r) {
fb3d9e97 5144 r600_blit_fini(rdev);
27cd7769 5145 rdev->asic->copy.copy = NULL;
d7ccd8fc 5146 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
bcc1c2a1
AD		 5147 }
5148
2948f5e6
AD		 5149 /* allocate rlc buffers */
5150 if (rdev->flags & RADEON_IS_IGP) {
5151 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5152 rdev->rlc.reg_list_size = sumo_rlc_save_restore_register_list_size;
5153 rdev->rlc.cs_data = evergreen_cs_data;
5154 r = sumo_rlc_init(rdev);
5155 if (r) {
5156 DRM_ERROR("Failed to init rlc BOs!\n");
5157 return r;
5158 }
5159 }
5160
724c80e1
AD		 5161 /* allocate wb buffer */
5162 r = radeon_wb_init(rdev);
5163 if (r)
5164 return r;
5165
30eb77f4
JG		 5166 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5167 if (r) {
5168 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5169 return r;
5170 }
5171
233d1ad5
AD		 5172 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5173 if (r) {
5174 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5175 return r;
5176 }
5177
f2ba57b5
CK		 5178 r = rv770_uvd_resume(rdev);
5179 if (!r) {
5180 r = radeon_fence_driver_start_ring(rdev,
5181 R600_RING_TYPE_UVD_INDEX);
5182 if (r)
5183 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5184 }
5185
5186 if (r)
5187 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5188
bcc1c2a1 5189 /* Enable IRQ */
e49f3959
AH		 5190 if (!rdev->irq.installed) {
5191 r = radeon_irq_kms_init(rdev);
5192 if (r)
5193 return r;
5194 }
5195
bcc1c2a1
AD		 5196 r = r600_irq_init(rdev);
5197 if (r) {
5198 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5199 radeon_irq_kms_fini(rdev);
5200 return r;
5201 }
45f9a39b 5202 evergreen_irq_set(rdev);
bcc1c2a1 5203
f2ba57b5 5204 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5205 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a
AD		 5206 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5207 0, 0xfffff, RADEON_CP_PACKET2);
bcc1c2a1
AD		 5208 if (r)
5209 return r;
233d1ad5
AD		 5210
5211 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5212 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5213 DMA_RB_RPTR, DMA_RB_WPTR,
0fcb6155 5214 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5215 if (r)
5216 return r;
5217
bcc1c2a1
AD		 5218 r = evergreen_cp_load_microcode(rdev);
5219 if (r)
5220 return r;
fe251e2f 5221 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5222 if (r)
5223 return r;
5224 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5225 if (r)
5226 return r;
fe251e2f 5227
f2ba57b5
CK		 5228 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5229 if (ring->ring_size) {
5230 r = radeon_ring_init(rdev, ring, ring->ring_size,
5231 R600_WB_UVD_RPTR_OFFSET,
5232 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5233 0, 0xfffff, RADEON_CP_PACKET2);
5234 if (!r)
5235 r = r600_uvd_init(rdev);
5236
5237 if (r)
5238 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5239 }
5240
2898c348
CK		 5241 r = radeon_ib_pool_init(rdev);
5242 if (r) {
5243 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5244 return r;
2898c348 5245 }
b15ba512 5246
69d2ae57
RM		 5247 r = r600_audio_init(rdev);
5248 if (r) {
5249 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5250 return r;
5251 }
5252
bcc1c2a1
AD		 5253 return 0;
5254}
5255
5256int evergreen_resume(struct radeon_device *rdev)
5257{
5258 int r;
5259
86f5c9ed
AD		 5260 /* reset the asic, the gfx blocks are often in a bad state
5261 * after the driver is unloaded or after a resume
5262 */
5263 if (radeon_asic_reset(rdev))
5264 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5265 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5266 * posting will perform necessary task to bring back GPU into good
5267 * shape.
5268 */
5269 /* post card */
5270 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5271
d4788db3
AD		 5272 /* init golden registers */
5273 evergreen_init_golden_registers(rdev);
5274
b15ba512 5275 rdev->accel_working = true;
bcc1c2a1
AD		 5276 r = evergreen_startup(rdev);
5277 if (r) {
755d819e 5278 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5279 rdev->accel_working = false;
bcc1c2a1
AD		 5280 return r;
5281 }
fe251e2f 5282
bcc1c2a1
AD		 5283 return r;
5284
5285}
5286
5287int evergreen_suspend(struct radeon_device *rdev)
5288{
69d2ae57 5289 r600_audio_fini(rdev);
f2ba57b5 5290 radeon_uvd_suspend(rdev);
bcc1c2a1 5291 r700_cp_stop(rdev);
233d1ad5 5292 r600_dma_stop(rdev);
f2ba57b5 5293 r600_uvd_rbc_stop(rdev);
45f9a39b 5294 evergreen_irq_suspend(rdev);
724c80e1 5295 radeon_wb_disable(rdev);
bcc1c2a1 5296 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5297
5298 return 0;
5299}
5300
bcc1c2a1
AD		 5301/* Plan is to move initialization in that function and use
5302 * helper function so that radeon_device_init pretty much
5303 * do nothing more than calling asic specific function. This
5304 * should also allow to remove a bunch of callback function
5305 * like vram_info.
5306 */
5307int evergreen_init(struct radeon_device *rdev)
5308{
5309 int r;
5310
bcc1c2a1
AD		 5311 /* Read BIOS */
5312 if (!radeon_get_bios(rdev)) {
5313 if (ASIC_IS_AVIVO(rdev))
5314 return -EINVAL;
5315 }
5316 /* Must be an ATOMBIOS */
5317 if (!rdev->is_atom_bios) {
755d819e 5318 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5319 return -EINVAL;
5320 }
5321 r = radeon_atombios_init(rdev);
5322 if (r)
5323 return r;
86f5c9ed
AD		 5324 /* reset the asic, the gfx blocks are often in a bad state
5325 * after the driver is unloaded or after a resume
5326 */
5327 if (radeon_asic_reset(rdev))
5328 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5329 /* Post card if necessary */
fd909c37 5330 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5331 if (!rdev->bios) {
5332 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5333 return -EINVAL;
5334 }
5335 DRM_INFO("GPU not posted. posting now...\n");
5336 atom_asic_init(rdev->mode_info.atom_context);
5337 }
d4788db3
AD		 5338 /* init golden registers */
5339 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5340 /* Initialize scratch registers */
5341 r600_scratch_init(rdev);
5342 /* Initialize surface registers */
5343 radeon_surface_init(rdev);
5344 /* Initialize clocks */
5345 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5346 /* Fence driver */
5347 r = radeon_fence_driver_init(rdev);
5348 if (r)
5349 return r;
d594e46a
JG		 5350 /* initialize AGP */
5351 if (rdev->flags & RADEON_IS_AGP) {
5352 r = radeon_agp_init(rdev);
5353 if (r)
5354 radeon_agp_disable(rdev);
5355 }
5356 /* initialize memory controller */
bcc1c2a1
AD		 5357 r = evergreen_mc_init(rdev);
5358 if (r)
5359 return r;
5360 /* Memory manager */
5361 r = radeon_bo_init(rdev);
5362 if (r)
5363 return r;
45f9a39b 5364
e32eb50d
CK		 5365 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5366 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5367
233d1ad5
AD		 5368 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5369 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5370
f2ba57b5
CK		 5371 r = radeon_uvd_init(rdev);
5372 if (!r) {
5373 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5374 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5375 4096);
5376 }
5377
bcc1c2a1
AD		 5378 rdev->ih.ring_obj = NULL;
5379 r600_ih_ring_init(rdev, 64 * 1024);
5380
5381 r = r600_pcie_gart_init(rdev);
5382 if (r)
5383 return r;
0fcdb61e 5384
148a03bc 5385 rdev->accel_working = true;
bcc1c2a1
AD		 5386 r = evergreen_startup(rdev);
5387 if (r) {
fe251e2f
AD		 5388 dev_err(rdev->dev, "disabling GPU acceleration\n");
5389 r700_cp_fini(rdev);
233d1ad5 5390 r600_dma_fini(rdev);
fe251e2f 5391 r600_irq_fini(rdev);
2948f5e6
AD		 5392 if (rdev->flags & RADEON_IS_IGP)
5393 sumo_rlc_fini(rdev);
724c80e1 5394 radeon_wb_fini(rdev);
2898c348 5395 radeon_ib_pool_fini(rdev);
fe251e2f 5396 radeon_irq_kms_fini(rdev);
0fcdb61e 5397 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5398 rdev->accel_working = false;
5399 }
77e00f2e
AD		 5400
5401 /* Don't start up if the MC ucode is missing on BTC parts.
5402 * The default clocks and voltages before the MC ucode
5403 * is loaded are not suffient for advanced operations.
5404 */
5405 if (ASIC_IS_DCE5(rdev)) {
5406 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5407 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5408 return -EINVAL;
5409 }
5410 }
5411
bcc1c2a1
AD		 5412 return 0;
5413}
5414
5415void evergreen_fini(struct radeon_device *rdev)
5416{
69d2ae57 5417 r600_audio_fini(rdev);
fb3d9e97 5418 r600_blit_fini(rdev);
45f9a39b 5419 r700_cp_fini(rdev);
233d1ad5 5420 r600_dma_fini(rdev);
bcc1c2a1 5421 r600_irq_fini(rdev);
2948f5e6
AD		 5422 if (rdev->flags & RADEON_IS_IGP)
5423 sumo_rlc_fini(rdev);
724c80e1 5424 radeon_wb_fini(rdev);
2898c348 5425 radeon_ib_pool_fini(rdev);
bcc1c2a1 5426 radeon_irq_kms_fini(rdev);
bcc1c2a1 5427 evergreen_pcie_gart_fini(rdev);
f2ba57b5 5428 radeon_uvd_fini(rdev);
16cdf04d 5429 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5430 radeon_gem_fini(rdev);
5431 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5432 radeon_agp_fini(rdev);
5433 radeon_bo_fini(rdev);
5434 radeon_atombios_fini(rdev);
5435 kfree(rdev->bios);
5436 rdev->bios = NULL;
bcc1c2a1 5437}
9e46a48d 5438
b07759bf 5439void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5440{
7e0e4196 5441 u32 link_width_cntl, speed_cntl;
9e46a48d 5442
d42dd579
AD		 5443 if (radeon_pcie_gen2 == 0)
5444 return;
5445
9e46a48d
AD		 5446 if (rdev->flags & RADEON_IS_IGP)
5447 return;
5448
5449 if (!(rdev->flags & RADEON_IS_PCIE))
5450 return;
5451
5452 /* x2 cards have a special sequence */
5453 if (ASIC_IS_X2(rdev))
5454 return;
5455
7e0e4196
KSS		 5456 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5457 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5458 return;
5459
492d2b61 5460 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5461 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5462 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5463 return;
5464 }
5465
197bbb3d
DA		 5466 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5467
9e46a48d
AD		 5468 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5469 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5470
492d2b61 5471 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5472 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5473 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5474
492d2b61 5475 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5476 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5477 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5478
492d2b61 5479 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5480 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5481 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5482
492d2b61 5483 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5484 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5485 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5486
492d2b61 5487 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5488 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5489 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5490
5491 } else {
492d2b61 5492 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5493 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5494 if (1)
5495 link_width_cntl |= LC_UPCONFIGURE_DIS;
5496 else
5497 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5498 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5499 }
5500}
f52382d7
AD		 5501
5502void evergreen_program_aspm(struct radeon_device *rdev)
5503{
5504 u32 data, orig;
5505 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5506 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5507 /* fusion_platform = true
5508 * if the system is a fusion system
5509 * (APU or DGPU in a fusion system).
5510 * todo: check if the system is a fusion platform.
5511 */
5512 bool fusion_platform = false;
5513
5514 if (!(rdev->flags & RADEON_IS_PCIE))
5515 return;
5516
5517 switch (rdev->family) {
5518 case CHIP_CYPRESS:
5519 case CHIP_HEMLOCK:
5520 case CHIP_JUNIPER:
5521 case CHIP_REDWOOD:
5522 case CHIP_CEDAR:
5523 case CHIP_SUMO:
5524 case CHIP_SUMO2:
5525 case CHIP_PALM:
5526 case CHIP_ARUBA:
5527 disable_l0s = true;
5528 break;
5529 default:
5530 disable_l0s = false;
5531 break;
5532 }
5533
5534 if (rdev->flags & RADEON_IS_IGP)
5535 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5536
5537 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5538 if (fusion_platform)
5539 data &= ~MULTI_PIF;
5540 else
5541 data |= MULTI_PIF;
5542 if (data != orig)
5543 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5544
5545 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5546 if (fusion_platform)
5547 data &= ~MULTI_PIF;
5548 else
5549 data |= MULTI_PIF;
5550 if (data != orig)
5551 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5552
5553 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5554 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5555 if (!disable_l0s) {
5556 if (rdev->family >= CHIP_BARTS)
5557 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5558 else
5559 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5560 }
5561
5562 if (!disable_l1) {
5563 if (rdev->family >= CHIP_BARTS)
5564 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5565 else
5566 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5567
5568 if (!disable_plloff_in_l1) {
5569 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5570 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5571 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5572 if (data != orig)
5573 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5574
5575 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5576 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5577 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5578 if (data != orig)
5579 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5580
5581 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5582 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5583 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5584 if (data != orig)
5585 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5586
5587 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5588 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5589 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5590 if (data != orig)
5591 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5592
5593 if (rdev->family >= CHIP_BARTS) {
5594 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5595 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5596 data |= PLL_RAMP_UP_TIME_0(4);
5597 if (data != orig)
5598 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5599
5600 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5601 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5602 data |= PLL_RAMP_UP_TIME_1(4);
5603 if (data != orig)
5604 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5605
5606 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5607 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5608 data |= PLL_RAMP_UP_TIME_0(4);
5609 if (data != orig)
5610 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5611
5612 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5613 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5614 data |= PLL_RAMP_UP_TIME_1(4);
5615 if (data != orig)
5616 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5617 }
5618
5619 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5620 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5621 data |= LC_DYN_LANES_PWR_STATE(3);
5622 if (data != orig)
5623 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5624
5625 if (rdev->family >= CHIP_BARTS) {
5626 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5627 data &= ~LS2_EXIT_TIME_MASK;
5628 data |= LS2_EXIT_TIME(1);
5629 if (data != orig)
5630 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5631
5632 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5633 data &= ~LS2_EXIT_TIME_MASK;
5634 data |= LS2_EXIT_TIME(1);
5635 if (data != orig)
5636 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5637 }
5638 }
5639 }
5640
5641 /* evergreen parts only */
5642 if (rdev->family < CHIP_BARTS)
5643 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5644
5645 if (pcie_lc_cntl != pcie_lc_cntl_old)
5646 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5647}
Linux 6.x block layer and io_uring tree(s)