projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: convert SI,CIK to use sumo_rlc functions
[linux-2.6-block.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
5a0e3ad6 26#include <linux/slab.h>
760285e7 27#include <drm/drmP.h>
bcc1c2a1 28#include "radeon.h"
e6990375 29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
4a15903d
AD		 38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
2948f5e6
AD		 48#include "clearstate_evergreen.h"
49
1fd11777 50static const u32 sumo_rlc_save_restore_register_list[] =
2948f5e6
AD		 51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
2948f5e6 134
bcc1c2a1
AD		 135static void evergreen_gpu_init(struct radeon_device *rdev);
136void evergreen_fini(struct radeon_device *rdev);
b07759bf 137void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 138void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 139extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
54e2e49c
AD		 141extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
bcc1c2a1 143
d4788db3
AD		 144static const u32 evergreen_golden_registers[] =
145{
146 0x3f90, 0xffff0000, 0xff000000,
147 0x9148, 0xffff0000, 0xff000000,
148 0x3f94, 0xffff0000, 0xff000000,
149 0x914c, 0xffff0000, 0xff000000,
150 0x9b7c, 0xffffffff, 0x00000000,
151 0x8a14, 0xffffffff, 0x00000007,
152 0x8b10, 0xffffffff, 0x00000000,
153 0x960c, 0xffffffff, 0x54763210,
154 0x88c4, 0xffffffff, 0x000000c2,
155 0x88d4, 0xffffffff, 0x00000010,
156 0x8974, 0xffffffff, 0x00000000,
157 0xc78, 0x00000080, 0x00000080,
158 0x5eb4, 0xffffffff, 0x00000002,
159 0x5e78, 0xffffffff, 0x001000f0,
160 0x6104, 0x01000300, 0x00000000,
161 0x5bc0, 0x00300000, 0x00000000,
162 0x7030, 0xffffffff, 0x00000011,
163 0x7c30, 0xffffffff, 0x00000011,
164 0x10830, 0xffffffff, 0x00000011,
165 0x11430, 0xffffffff, 0x00000011,
166 0x12030, 0xffffffff, 0x00000011,
167 0x12c30, 0xffffffff, 0x00000011,
168 0xd02c, 0xffffffff, 0x08421000,
169 0x240c, 0xffffffff, 0x00000380,
170 0x8b24, 0xffffffff, 0x00ff0fff,
171 0x28a4c, 0x06000000, 0x06000000,
172 0x10c, 0x00000001, 0x00000001,
173 0x8d00, 0xffffffff, 0x100e4848,
174 0x8d04, 0xffffffff, 0x00164745,
175 0x8c00, 0xffffffff, 0xe4000003,
176 0x8c04, 0xffffffff, 0x40600060,
177 0x8c08, 0xffffffff, 0x001c001c,
178 0x8cf0, 0xffffffff, 0x08e00620,
179 0x8c20, 0xffffffff, 0x00800080,
180 0x8c24, 0xffffffff, 0x00800080,
181 0x8c18, 0xffffffff, 0x20202078,
182 0x8c1c, 0xffffffff, 0x00001010,
183 0x28350, 0xffffffff, 0x00000000,
184 0xa008, 0xffffffff, 0x00010000,
185 0x5cc, 0xffffffff, 0x00000001,
186 0x9508, 0xffffffff, 0x00000002,
187 0x913c, 0x0000000f, 0x0000000a
188};
189
190static const u32 evergreen_golden_registers2[] =
191{
192 0x2f4c, 0xffffffff, 0x00000000,
193 0x54f4, 0xffffffff, 0x00000000,
194 0x54f0, 0xffffffff, 0x00000000,
195 0x5498, 0xffffffff, 0x00000000,
196 0x549c, 0xffffffff, 0x00000000,
197 0x5494, 0xffffffff, 0x00000000,
198 0x53cc, 0xffffffff, 0x00000000,
199 0x53c8, 0xffffffff, 0x00000000,
200 0x53c4, 0xffffffff, 0x00000000,
201 0x53c0, 0xffffffff, 0x00000000,
202 0x53bc, 0xffffffff, 0x00000000,
203 0x53b8, 0xffffffff, 0x00000000,
204 0x53b4, 0xffffffff, 0x00000000,
205 0x53b0, 0xffffffff, 0x00000000
206};
207
208static const u32 cypress_mgcg_init[] =
209{
210 0x802c, 0xffffffff, 0xc0000000,
211 0x5448, 0xffffffff, 0x00000100,
212 0x55e4, 0xffffffff, 0x00000100,
213 0x160c, 0xffffffff, 0x00000100,
214 0x5644, 0xffffffff, 0x00000100,
215 0xc164, 0xffffffff, 0x00000100,
216 0x8a18, 0xffffffff, 0x00000100,
217 0x897c, 0xffffffff, 0x06000100,
218 0x8b28, 0xffffffff, 0x00000100,
219 0x9144, 0xffffffff, 0x00000100,
220 0x9a60, 0xffffffff, 0x00000100,
221 0x9868, 0xffffffff, 0x00000100,
222 0x8d58, 0xffffffff, 0x00000100,
223 0x9510, 0xffffffff, 0x00000100,
224 0x949c, 0xffffffff, 0x00000100,
225 0x9654, 0xffffffff, 0x00000100,
226 0x9030, 0xffffffff, 0x00000100,
227 0x9034, 0xffffffff, 0x00000100,
228 0x9038, 0xffffffff, 0x00000100,
229 0x903c, 0xffffffff, 0x00000100,
230 0x9040, 0xffffffff, 0x00000100,
231 0xa200, 0xffffffff, 0x00000100,
232 0xa204, 0xffffffff, 0x00000100,
233 0xa208, 0xffffffff, 0x00000100,
234 0xa20c, 0xffffffff, 0x00000100,
235 0x971c, 0xffffffff, 0x00000100,
236 0x977c, 0xffffffff, 0x00000100,
237 0x3f80, 0xffffffff, 0x00000100,
238 0xa210, 0xffffffff, 0x00000100,
239 0xa214, 0xffffffff, 0x00000100,
240 0x4d8, 0xffffffff, 0x00000100,
241 0x9784, 0xffffffff, 0x00000100,
242 0x9698, 0xffffffff, 0x00000100,
243 0x4d4, 0xffffffff, 0x00000200,
244 0x30cc, 0xffffffff, 0x00000100,
245 0xd0c0, 0xffffffff, 0xff000100,
246 0x802c, 0xffffffff, 0x40000000,
247 0x915c, 0xffffffff, 0x00010000,
248 0x9160, 0xffffffff, 0x00030002,
249 0x9178, 0xffffffff, 0x00070000,
250 0x917c, 0xffffffff, 0x00030002,
251 0x9180, 0xffffffff, 0x00050004,
252 0x918c, 0xffffffff, 0x00010006,
253 0x9190, 0xffffffff, 0x00090008,
254 0x9194, 0xffffffff, 0x00070000,
255 0x9198, 0xffffffff, 0x00030002,
256 0x919c, 0xffffffff, 0x00050004,
257 0x91a8, 0xffffffff, 0x00010006,
258 0x91ac, 0xffffffff, 0x00090008,
259 0x91b0, 0xffffffff, 0x00070000,
260 0x91b4, 0xffffffff, 0x00030002,
261 0x91b8, 0xffffffff, 0x00050004,
262 0x91c4, 0xffffffff, 0x00010006,
263 0x91c8, 0xffffffff, 0x00090008,
264 0x91cc, 0xffffffff, 0x00070000,
265 0x91d0, 0xffffffff, 0x00030002,
266 0x91d4, 0xffffffff, 0x00050004,
267 0x91e0, 0xffffffff, 0x00010006,
268 0x91e4, 0xffffffff, 0x00090008,
269 0x91e8, 0xffffffff, 0x00000000,
270 0x91ec, 0xffffffff, 0x00070000,
271 0x91f0, 0xffffffff, 0x00030002,
272 0x91f4, 0xffffffff, 0x00050004,
273 0x9200, 0xffffffff, 0x00010006,
274 0x9204, 0xffffffff, 0x00090008,
275 0x9208, 0xffffffff, 0x00070000,
276 0x920c, 0xffffffff, 0x00030002,
277 0x9210, 0xffffffff, 0x00050004,
278 0x921c, 0xffffffff, 0x00010006,
279 0x9220, 0xffffffff, 0x00090008,
280 0x9224, 0xffffffff, 0x00070000,
281 0x9228, 0xffffffff, 0x00030002,
282 0x922c, 0xffffffff, 0x00050004,
283 0x9238, 0xffffffff, 0x00010006,
284 0x923c, 0xffffffff, 0x00090008,
285 0x9240, 0xffffffff, 0x00070000,
286 0x9244, 0xffffffff, 0x00030002,
287 0x9248, 0xffffffff, 0x00050004,
288 0x9254, 0xffffffff, 0x00010006,
289 0x9258, 0xffffffff, 0x00090008,
290 0x925c, 0xffffffff, 0x00070000,
291 0x9260, 0xffffffff, 0x00030002,
292 0x9264, 0xffffffff, 0x00050004,
293 0x9270, 0xffffffff, 0x00010006,
294 0x9274, 0xffffffff, 0x00090008,
295 0x9278, 0xffffffff, 0x00070000,
296 0x927c, 0xffffffff, 0x00030002,
297 0x9280, 0xffffffff, 0x00050004,
298 0x928c, 0xffffffff, 0x00010006,
299 0x9290, 0xffffffff, 0x00090008,
300 0x9294, 0xffffffff, 0x00000000,
301 0x929c, 0xffffffff, 0x00000001,
302 0x802c, 0xffffffff, 0x40010000,
303 0x915c, 0xffffffff, 0x00010000,
304 0x9160, 0xffffffff, 0x00030002,
305 0x9178, 0xffffffff, 0x00070000,
306 0x917c, 0xffffffff, 0x00030002,
307 0x9180, 0xffffffff, 0x00050004,
308 0x918c, 0xffffffff, 0x00010006,
309 0x9190, 0xffffffff, 0x00090008,
310 0x9194, 0xffffffff, 0x00070000,
311 0x9198, 0xffffffff, 0x00030002,
312 0x919c, 0xffffffff, 0x00050004,
313 0x91a8, 0xffffffff, 0x00010006,
314 0x91ac, 0xffffffff, 0x00090008,
315 0x91b0, 0xffffffff, 0x00070000,
316 0x91b4, 0xffffffff, 0x00030002,
317 0x91b8, 0xffffffff, 0x00050004,
318 0x91c4, 0xffffffff, 0x00010006,
319 0x91c8, 0xffffffff, 0x00090008,
320 0x91cc, 0xffffffff, 0x00070000,
321 0x91d0, 0xffffffff, 0x00030002,
322 0x91d4, 0xffffffff, 0x00050004,
323 0x91e0, 0xffffffff, 0x00010006,
324 0x91e4, 0xffffffff, 0x00090008,
325 0x91e8, 0xffffffff, 0x00000000,
326 0x91ec, 0xffffffff, 0x00070000,
327 0x91f0, 0xffffffff, 0x00030002,
328 0x91f4, 0xffffffff, 0x00050004,
329 0x9200, 0xffffffff, 0x00010006,
330 0x9204, 0xffffffff, 0x00090008,
331 0x9208, 0xffffffff, 0x00070000,
332 0x920c, 0xffffffff, 0x00030002,
333 0x9210, 0xffffffff, 0x00050004,
334 0x921c, 0xffffffff, 0x00010006,
335 0x9220, 0xffffffff, 0x00090008,
336 0x9224, 0xffffffff, 0x00070000,
337 0x9228, 0xffffffff, 0x00030002,
338 0x922c, 0xffffffff, 0x00050004,
339 0x9238, 0xffffffff, 0x00010006,
340 0x923c, 0xffffffff, 0x00090008,
341 0x9240, 0xffffffff, 0x00070000,
342 0x9244, 0xffffffff, 0x00030002,
343 0x9248, 0xffffffff, 0x00050004,
344 0x9254, 0xffffffff, 0x00010006,
345 0x9258, 0xffffffff, 0x00090008,
346 0x925c, 0xffffffff, 0x00070000,
347 0x9260, 0xffffffff, 0x00030002,
348 0x9264, 0xffffffff, 0x00050004,
349 0x9270, 0xffffffff, 0x00010006,
350 0x9274, 0xffffffff, 0x00090008,
351 0x9278, 0xffffffff, 0x00070000,
352 0x927c, 0xffffffff, 0x00030002,
353 0x9280, 0xffffffff, 0x00050004,
354 0x928c, 0xffffffff, 0x00010006,
355 0x9290, 0xffffffff, 0x00090008,
356 0x9294, 0xffffffff, 0x00000000,
357 0x929c, 0xffffffff, 0x00000001,
358 0x802c, 0xffffffff, 0xc0000000
359};
360
361static const u32 redwood_mgcg_init[] =
362{
363 0x802c, 0xffffffff, 0xc0000000,
364 0x5448, 0xffffffff, 0x00000100,
365 0x55e4, 0xffffffff, 0x00000100,
366 0x160c, 0xffffffff, 0x00000100,
367 0x5644, 0xffffffff, 0x00000100,
368 0xc164, 0xffffffff, 0x00000100,
369 0x8a18, 0xffffffff, 0x00000100,
370 0x897c, 0xffffffff, 0x06000100,
371 0x8b28, 0xffffffff, 0x00000100,
372 0x9144, 0xffffffff, 0x00000100,
373 0x9a60, 0xffffffff, 0x00000100,
374 0x9868, 0xffffffff, 0x00000100,
375 0x8d58, 0xffffffff, 0x00000100,
376 0x9510, 0xffffffff, 0x00000100,
377 0x949c, 0xffffffff, 0x00000100,
378 0x9654, 0xffffffff, 0x00000100,
379 0x9030, 0xffffffff, 0x00000100,
380 0x9034, 0xffffffff, 0x00000100,
381 0x9038, 0xffffffff, 0x00000100,
382 0x903c, 0xffffffff, 0x00000100,
383 0x9040, 0xffffffff, 0x00000100,
384 0xa200, 0xffffffff, 0x00000100,
385 0xa204, 0xffffffff, 0x00000100,
386 0xa208, 0xffffffff, 0x00000100,
387 0xa20c, 0xffffffff, 0x00000100,
388 0x971c, 0xffffffff, 0x00000100,
389 0x977c, 0xffffffff, 0x00000100,
390 0x3f80, 0xffffffff, 0x00000100,
391 0xa210, 0xffffffff, 0x00000100,
392 0xa214, 0xffffffff, 0x00000100,
393 0x4d8, 0xffffffff, 0x00000100,
394 0x9784, 0xffffffff, 0x00000100,
395 0x9698, 0xffffffff, 0x00000100,
396 0x4d4, 0xffffffff, 0x00000200,
397 0x30cc, 0xffffffff, 0x00000100,
398 0xd0c0, 0xffffffff, 0xff000100,
399 0x802c, 0xffffffff, 0x40000000,
400 0x915c, 0xffffffff, 0x00010000,
401 0x9160, 0xffffffff, 0x00030002,
402 0x9178, 0xffffffff, 0x00070000,
403 0x917c, 0xffffffff, 0x00030002,
404 0x9180, 0xffffffff, 0x00050004,
405 0x918c, 0xffffffff, 0x00010006,
406 0x9190, 0xffffffff, 0x00090008,
407 0x9194, 0xffffffff, 0x00070000,
408 0x9198, 0xffffffff, 0x00030002,
409 0x919c, 0xffffffff, 0x00050004,
410 0x91a8, 0xffffffff, 0x00010006,
411 0x91ac, 0xffffffff, 0x00090008,
412 0x91b0, 0xffffffff, 0x00070000,
413 0x91b4, 0xffffffff, 0x00030002,
414 0x91b8, 0xffffffff, 0x00050004,
415 0x91c4, 0xffffffff, 0x00010006,
416 0x91c8, 0xffffffff, 0x00090008,
417 0x91cc, 0xffffffff, 0x00070000,
418 0x91d0, 0xffffffff, 0x00030002,
419 0x91d4, 0xffffffff, 0x00050004,
420 0x91e0, 0xffffffff, 0x00010006,
421 0x91e4, 0xffffffff, 0x00090008,
422 0x91e8, 0xffffffff, 0x00000000,
423 0x91ec, 0xffffffff, 0x00070000,
424 0x91f0, 0xffffffff, 0x00030002,
425 0x91f4, 0xffffffff, 0x00050004,
426 0x9200, 0xffffffff, 0x00010006,
427 0x9204, 0xffffffff, 0x00090008,
428 0x9294, 0xffffffff, 0x00000000,
429 0x929c, 0xffffffff, 0x00000001,
430 0x802c, 0xffffffff, 0xc0000000
431};
432
433static const u32 cedar_golden_registers[] =
434{
435 0x3f90, 0xffff0000, 0xff000000,
436 0x9148, 0xffff0000, 0xff000000,
437 0x3f94, 0xffff0000, 0xff000000,
438 0x914c, 0xffff0000, 0xff000000,
439 0x9b7c, 0xffffffff, 0x00000000,
440 0x8a14, 0xffffffff, 0x00000007,
441 0x8b10, 0xffffffff, 0x00000000,
442 0x960c, 0xffffffff, 0x54763210,
443 0x88c4, 0xffffffff, 0x000000c2,
444 0x88d4, 0xffffffff, 0x00000000,
445 0x8974, 0xffffffff, 0x00000000,
446 0xc78, 0x00000080, 0x00000080,
447 0x5eb4, 0xffffffff, 0x00000002,
448 0x5e78, 0xffffffff, 0x001000f0,
449 0x6104, 0x01000300, 0x00000000,
450 0x5bc0, 0x00300000, 0x00000000,
451 0x7030, 0xffffffff, 0x00000011,
452 0x7c30, 0xffffffff, 0x00000011,
453 0x10830, 0xffffffff, 0x00000011,
454 0x11430, 0xffffffff, 0x00000011,
455 0xd02c, 0xffffffff, 0x08421000,
456 0x240c, 0xffffffff, 0x00000380,
457 0x8b24, 0xffffffff, 0x00ff0fff,
458 0x28a4c, 0x06000000, 0x06000000,
459 0x10c, 0x00000001, 0x00000001,
460 0x8d00, 0xffffffff, 0x100e4848,
461 0x8d04, 0xffffffff, 0x00164745,
462 0x8c00, 0xffffffff, 0xe4000003,
463 0x8c04, 0xffffffff, 0x40600060,
464 0x8c08, 0xffffffff, 0x001c001c,
465 0x8cf0, 0xffffffff, 0x08e00410,
466 0x8c20, 0xffffffff, 0x00800080,
467 0x8c24, 0xffffffff, 0x00800080,
468 0x8c18, 0xffffffff, 0x20202078,
469 0x8c1c, 0xffffffff, 0x00001010,
470 0x28350, 0xffffffff, 0x00000000,
471 0xa008, 0xffffffff, 0x00010000,
472 0x5cc, 0xffffffff, 0x00000001,
473 0x9508, 0xffffffff, 0x00000002
474};
475
476static const u32 cedar_mgcg_init[] =
477{
478 0x802c, 0xffffffff, 0xc0000000,
479 0x5448, 0xffffffff, 0x00000100,
480 0x55e4, 0xffffffff, 0x00000100,
481 0x160c, 0xffffffff, 0x00000100,
482 0x5644, 0xffffffff, 0x00000100,
483 0xc164, 0xffffffff, 0x00000100,
484 0x8a18, 0xffffffff, 0x00000100,
485 0x897c, 0xffffffff, 0x06000100,
486 0x8b28, 0xffffffff, 0x00000100,
487 0x9144, 0xffffffff, 0x00000100,
488 0x9a60, 0xffffffff, 0x00000100,
489 0x9868, 0xffffffff, 0x00000100,
490 0x8d58, 0xffffffff, 0x00000100,
491 0x9510, 0xffffffff, 0x00000100,
492 0x949c, 0xffffffff, 0x00000100,
493 0x9654, 0xffffffff, 0x00000100,
494 0x9030, 0xffffffff, 0x00000100,
495 0x9034, 0xffffffff, 0x00000100,
496 0x9038, 0xffffffff, 0x00000100,
497 0x903c, 0xffffffff, 0x00000100,
498 0x9040, 0xffffffff, 0x00000100,
499 0xa200, 0xffffffff, 0x00000100,
500 0xa204, 0xffffffff, 0x00000100,
501 0xa208, 0xffffffff, 0x00000100,
502 0xa20c, 0xffffffff, 0x00000100,
503 0x971c, 0xffffffff, 0x00000100,
504 0x977c, 0xffffffff, 0x00000100,
505 0x3f80, 0xffffffff, 0x00000100,
506 0xa210, 0xffffffff, 0x00000100,
507 0xa214, 0xffffffff, 0x00000100,
508 0x4d8, 0xffffffff, 0x00000100,
509 0x9784, 0xffffffff, 0x00000100,
510 0x9698, 0xffffffff, 0x00000100,
511 0x4d4, 0xffffffff, 0x00000200,
512 0x30cc, 0xffffffff, 0x00000100,
513 0xd0c0, 0xffffffff, 0xff000100,
514 0x802c, 0xffffffff, 0x40000000,
515 0x915c, 0xffffffff, 0x00010000,
516 0x9178, 0xffffffff, 0x00050000,
517 0x917c, 0xffffffff, 0x00030002,
518 0x918c, 0xffffffff, 0x00010004,
519 0x9190, 0xffffffff, 0x00070006,
520 0x9194, 0xffffffff, 0x00050000,
521 0x9198, 0xffffffff, 0x00030002,
522 0x91a8, 0xffffffff, 0x00010004,
523 0x91ac, 0xffffffff, 0x00070006,
524 0x91e8, 0xffffffff, 0x00000000,
525 0x9294, 0xffffffff, 0x00000000,
526 0x929c, 0xffffffff, 0x00000001,
527 0x802c, 0xffffffff, 0xc0000000
528};
529
530static const u32 juniper_mgcg_init[] =
531{
532 0x802c, 0xffffffff, 0xc0000000,
533 0x5448, 0xffffffff, 0x00000100,
534 0x55e4, 0xffffffff, 0x00000100,
535 0x160c, 0xffffffff, 0x00000100,
536 0x5644, 0xffffffff, 0x00000100,
537 0xc164, 0xffffffff, 0x00000100,
538 0x8a18, 0xffffffff, 0x00000100,
539 0x897c, 0xffffffff, 0x06000100,
540 0x8b28, 0xffffffff, 0x00000100,
541 0x9144, 0xffffffff, 0x00000100,
542 0x9a60, 0xffffffff, 0x00000100,
543 0x9868, 0xffffffff, 0x00000100,
544 0x8d58, 0xffffffff, 0x00000100,
545 0x9510, 0xffffffff, 0x00000100,
546 0x949c, 0xffffffff, 0x00000100,
547 0x9654, 0xffffffff, 0x00000100,
548 0x9030, 0xffffffff, 0x00000100,
549 0x9034, 0xffffffff, 0x00000100,
550 0x9038, 0xffffffff, 0x00000100,
551 0x903c, 0xffffffff, 0x00000100,
552 0x9040, 0xffffffff, 0x00000100,
553 0xa200, 0xffffffff, 0x00000100,
554 0xa204, 0xffffffff, 0x00000100,
555 0xa208, 0xffffffff, 0x00000100,
556 0xa20c, 0xffffffff, 0x00000100,
557 0x971c, 0xffffffff, 0x00000100,
558 0xd0c0, 0xffffffff, 0xff000100,
559 0x802c, 0xffffffff, 0x40000000,
560 0x915c, 0xffffffff, 0x00010000,
561 0x9160, 0xffffffff, 0x00030002,
562 0x9178, 0xffffffff, 0x00070000,
563 0x917c, 0xffffffff, 0x00030002,
564 0x9180, 0xffffffff, 0x00050004,
565 0x918c, 0xffffffff, 0x00010006,
566 0x9190, 0xffffffff, 0x00090008,
567 0x9194, 0xffffffff, 0x00070000,
568 0x9198, 0xffffffff, 0x00030002,
569 0x919c, 0xffffffff, 0x00050004,
570 0x91a8, 0xffffffff, 0x00010006,
571 0x91ac, 0xffffffff, 0x00090008,
572 0x91b0, 0xffffffff, 0x00070000,
573 0x91b4, 0xffffffff, 0x00030002,
574 0x91b8, 0xffffffff, 0x00050004,
575 0x91c4, 0xffffffff, 0x00010006,
576 0x91c8, 0xffffffff, 0x00090008,
577 0x91cc, 0xffffffff, 0x00070000,
578 0x91d0, 0xffffffff, 0x00030002,
579 0x91d4, 0xffffffff, 0x00050004,
580 0x91e0, 0xffffffff, 0x00010006,
581 0x91e4, 0xffffffff, 0x00090008,
582 0x91e8, 0xffffffff, 0x00000000,
583 0x91ec, 0xffffffff, 0x00070000,
584 0x91f0, 0xffffffff, 0x00030002,
585 0x91f4, 0xffffffff, 0x00050004,
586 0x9200, 0xffffffff, 0x00010006,
587 0x9204, 0xffffffff, 0x00090008,
588 0x9208, 0xffffffff, 0x00070000,
589 0x920c, 0xffffffff, 0x00030002,
590 0x9210, 0xffffffff, 0x00050004,
591 0x921c, 0xffffffff, 0x00010006,
592 0x9220, 0xffffffff, 0x00090008,
593 0x9224, 0xffffffff, 0x00070000,
594 0x9228, 0xffffffff, 0x00030002,
595 0x922c, 0xffffffff, 0x00050004,
596 0x9238, 0xffffffff, 0x00010006,
597 0x923c, 0xffffffff, 0x00090008,
598 0x9240, 0xffffffff, 0x00070000,
599 0x9244, 0xffffffff, 0x00030002,
600 0x9248, 0xffffffff, 0x00050004,
601 0x9254, 0xffffffff, 0x00010006,
602 0x9258, 0xffffffff, 0x00090008,
603 0x925c, 0xffffffff, 0x00070000,
604 0x9260, 0xffffffff, 0x00030002,
605 0x9264, 0xffffffff, 0x00050004,
606 0x9270, 0xffffffff, 0x00010006,
607 0x9274, 0xffffffff, 0x00090008,
608 0x9278, 0xffffffff, 0x00070000,
609 0x927c, 0xffffffff, 0x00030002,
610 0x9280, 0xffffffff, 0x00050004,
611 0x928c, 0xffffffff, 0x00010006,
612 0x9290, 0xffffffff, 0x00090008,
613 0x9294, 0xffffffff, 0x00000000,
614 0x929c, 0xffffffff, 0x00000001,
615 0x802c, 0xffffffff, 0xc0000000,
616 0x977c, 0xffffffff, 0x00000100,
617 0x3f80, 0xffffffff, 0x00000100,
618 0xa210, 0xffffffff, 0x00000100,
619 0xa214, 0xffffffff, 0x00000100,
620 0x4d8, 0xffffffff, 0x00000100,
621 0x9784, 0xffffffff, 0x00000100,
622 0x9698, 0xffffffff, 0x00000100,
623 0x4d4, 0xffffffff, 0x00000200,
624 0x30cc, 0xffffffff, 0x00000100,
625 0x802c, 0xffffffff, 0xc0000000
626};
627
628static const u32 supersumo_golden_registers[] =
629{
630 0x5eb4, 0xffffffff, 0x00000002,
631 0x5cc, 0xffffffff, 0x00000001,
632 0x7030, 0xffffffff, 0x00000011,
633 0x7c30, 0xffffffff, 0x00000011,
634 0x6104, 0x01000300, 0x00000000,
635 0x5bc0, 0x00300000, 0x00000000,
636 0x8c04, 0xffffffff, 0x40600060,
637 0x8c08, 0xffffffff, 0x001c001c,
638 0x8c20, 0xffffffff, 0x00800080,
639 0x8c24, 0xffffffff, 0x00800080,
640 0x8c18, 0xffffffff, 0x20202078,
641 0x8c1c, 0xffffffff, 0x00001010,
642 0x918c, 0xffffffff, 0x00010006,
643 0x91a8, 0xffffffff, 0x00010006,
644 0x91c4, 0xffffffff, 0x00010006,
645 0x91e0, 0xffffffff, 0x00010006,
646 0x9200, 0xffffffff, 0x00010006,
647 0x9150, 0xffffffff, 0x6e944040,
648 0x917c, 0xffffffff, 0x00030002,
649 0x9180, 0xffffffff, 0x00050004,
650 0x9198, 0xffffffff, 0x00030002,
651 0x919c, 0xffffffff, 0x00050004,
652 0x91b4, 0xffffffff, 0x00030002,
653 0x91b8, 0xffffffff, 0x00050004,
654 0x91d0, 0xffffffff, 0x00030002,
655 0x91d4, 0xffffffff, 0x00050004,
656 0x91f0, 0xffffffff, 0x00030002,
657 0x91f4, 0xffffffff, 0x00050004,
658 0x915c, 0xffffffff, 0x00010000,
659 0x9160, 0xffffffff, 0x00030002,
660 0x3f90, 0xffff0000, 0xff000000,
661 0x9178, 0xffffffff, 0x00070000,
662 0x9194, 0xffffffff, 0x00070000,
663 0x91b0, 0xffffffff, 0x00070000,
664 0x91cc, 0xffffffff, 0x00070000,
665 0x91ec, 0xffffffff, 0x00070000,
666 0x9148, 0xffff0000, 0xff000000,
667 0x9190, 0xffffffff, 0x00090008,
668 0x91ac, 0xffffffff, 0x00090008,
669 0x91c8, 0xffffffff, 0x00090008,
670 0x91e4, 0xffffffff, 0x00090008,
671 0x9204, 0xffffffff, 0x00090008,
672 0x3f94, 0xffff0000, 0xff000000,
673 0x914c, 0xffff0000, 0xff000000,
674 0x929c, 0xffffffff, 0x00000001,
675 0x8a18, 0xffffffff, 0x00000100,
676 0x8b28, 0xffffffff, 0x00000100,
677 0x9144, 0xffffffff, 0x00000100,
678 0x5644, 0xffffffff, 0x00000100,
679 0x9b7c, 0xffffffff, 0x00000000,
680 0x8030, 0xffffffff, 0x0000100a,
681 0x8a14, 0xffffffff, 0x00000007,
682 0x8b24, 0xffffffff, 0x00ff0fff,
683 0x8b10, 0xffffffff, 0x00000000,
684 0x28a4c, 0x06000000, 0x06000000,
685 0x4d8, 0xffffffff, 0x00000100,
686 0x913c, 0xffff000f, 0x0100000a,
687 0x960c, 0xffffffff, 0x54763210,
688 0x88c4, 0xffffffff, 0x000000c2,
689 0x88d4, 0xffffffff, 0x00000010,
690 0x8974, 0xffffffff, 0x00000000,
691 0xc78, 0x00000080, 0x00000080,
692 0x5e78, 0xffffffff, 0x001000f0,
693 0xd02c, 0xffffffff, 0x08421000,
694 0xa008, 0xffffffff, 0x00010000,
695 0x8d00, 0xffffffff, 0x100e4848,
696 0x8d04, 0xffffffff, 0x00164745,
697 0x8c00, 0xffffffff, 0xe4000003,
698 0x8cf0, 0x1fffffff, 0x08e00620,
699 0x28350, 0xffffffff, 0x00000000,
700 0x9508, 0xffffffff, 0x00000002
701};
702
703static const u32 sumo_golden_registers[] =
704{
705 0x900c, 0x00ffffff, 0x0017071f,
706 0x8c18, 0xffffffff, 0x10101060,
707 0x8c1c, 0xffffffff, 0x00001010,
708 0x8c30, 0x0000000f, 0x00000005,
709 0x9688, 0x0000000f, 0x00000007
710};
711
712static const u32 wrestler_golden_registers[] =
713{
714 0x5eb4, 0xffffffff, 0x00000002,
715 0x5cc, 0xffffffff, 0x00000001,
716 0x7030, 0xffffffff, 0x00000011,
717 0x7c30, 0xffffffff, 0x00000011,
718 0x6104, 0x01000300, 0x00000000,
719 0x5bc0, 0x00300000, 0x00000000,
720 0x918c, 0xffffffff, 0x00010006,
721 0x91a8, 0xffffffff, 0x00010006,
722 0x9150, 0xffffffff, 0x6e944040,
723 0x917c, 0xffffffff, 0x00030002,
724 0x9198, 0xffffffff, 0x00030002,
725 0x915c, 0xffffffff, 0x00010000,
726 0x3f90, 0xffff0000, 0xff000000,
727 0x9178, 0xffffffff, 0x00070000,
728 0x9194, 0xffffffff, 0x00070000,
729 0x9148, 0xffff0000, 0xff000000,
730 0x9190, 0xffffffff, 0x00090008,
731 0x91ac, 0xffffffff, 0x00090008,
732 0x3f94, 0xffff0000, 0xff000000,
733 0x914c, 0xffff0000, 0xff000000,
734 0x929c, 0xffffffff, 0x00000001,
735 0x8a18, 0xffffffff, 0x00000100,
736 0x8b28, 0xffffffff, 0x00000100,
737 0x9144, 0xffffffff, 0x00000100,
738 0x9b7c, 0xffffffff, 0x00000000,
739 0x8030, 0xffffffff, 0x0000100a,
740 0x8a14, 0xffffffff, 0x00000001,
741 0x8b24, 0xffffffff, 0x00ff0fff,
742 0x8b10, 0xffffffff, 0x00000000,
743 0x28a4c, 0x06000000, 0x06000000,
744 0x4d8, 0xffffffff, 0x00000100,
745 0x913c, 0xffff000f, 0x0100000a,
746 0x960c, 0xffffffff, 0x54763210,
747 0x88c4, 0xffffffff, 0x000000c2,
748 0x88d4, 0xffffffff, 0x00000010,
749 0x8974, 0xffffffff, 0x00000000,
750 0xc78, 0x00000080, 0x00000080,
751 0x5e78, 0xffffffff, 0x001000f0,
752 0xd02c, 0xffffffff, 0x08421000,
753 0xa008, 0xffffffff, 0x00010000,
754 0x8d00, 0xffffffff, 0x100e4848,
755 0x8d04, 0xffffffff, 0x00164745,
756 0x8c00, 0xffffffff, 0xe4000003,
757 0x8cf0, 0x1fffffff, 0x08e00410,
758 0x28350, 0xffffffff, 0x00000000,
759 0x9508, 0xffffffff, 0x00000002,
760 0x900c, 0xffffffff, 0x0017071f,
761 0x8c18, 0xffffffff, 0x10101060,
762 0x8c1c, 0xffffffff, 0x00001010
763};
764
765static const u32 barts_golden_registers[] =
766{
767 0x5eb4, 0xffffffff, 0x00000002,
768 0x5e78, 0x8f311ff1, 0x001000f0,
769 0x3f90, 0xffff0000, 0xff000000,
770 0x9148, 0xffff0000, 0xff000000,
771 0x3f94, 0xffff0000, 0xff000000,
772 0x914c, 0xffff0000, 0xff000000,
773 0xc78, 0x00000080, 0x00000080,
774 0xbd4, 0x70073777, 0x00010001,
775 0xd02c, 0xbfffff1f, 0x08421000,
776 0xd0b8, 0x03773777, 0x02011003,
777 0x5bc0, 0x00200000, 0x50100000,
778 0x98f8, 0x33773777, 0x02011003,
779 0x98fc, 0xffffffff, 0x76543210,
780 0x7030, 0x31000311, 0x00000011,
781 0x2f48, 0x00000007, 0x02011003,
782 0x6b28, 0x00000010, 0x00000012,
783 0x7728, 0x00000010, 0x00000012,
784 0x10328, 0x00000010, 0x00000012,
785 0x10f28, 0x00000010, 0x00000012,
786 0x11b28, 0x00000010, 0x00000012,
787 0x12728, 0x00000010, 0x00000012,
788 0x240c, 0x000007ff, 0x00000380,
789 0x8a14, 0xf000001f, 0x00000007,
790 0x8b24, 0x3fff3fff, 0x00ff0fff,
791 0x8b10, 0x0000ff0f, 0x00000000,
792 0x28a4c, 0x07ffffff, 0x06000000,
793 0x10c, 0x00000001, 0x00010003,
794 0xa02c, 0xffffffff, 0x0000009b,
795 0x913c, 0x0000000f, 0x0100000a,
796 0x8d00, 0xffff7f7f, 0x100e4848,
797 0x8d04, 0x00ffffff, 0x00164745,
798 0x8c00, 0xfffc0003, 0xe4000003,
799 0x8c04, 0xf8ff00ff, 0x40600060,
800 0x8c08, 0x00ff00ff, 0x001c001c,
801 0x8cf0, 0x1fff1fff, 0x08e00620,
802 0x8c20, 0x0fff0fff, 0x00800080,
803 0x8c24, 0x0fff0fff, 0x00800080,
804 0x8c18, 0xffffffff, 0x20202078,
805 0x8c1c, 0x0000ffff, 0x00001010,
806 0x28350, 0x00000f01, 0x00000000,
807 0x9508, 0x3700001f, 0x00000002,
808 0x960c, 0xffffffff, 0x54763210,
809 0x88c4, 0x001f3ae3, 0x000000c2,
810 0x88d4, 0x0000001f, 0x00000010,
811 0x8974, 0xffffffff, 0x00000000
812};
813
814static const u32 turks_golden_registers[] =
815{
816 0x5eb4, 0xffffffff, 0x00000002,
817 0x5e78, 0x8f311ff1, 0x001000f0,
818 0x8c8, 0x00003000, 0x00001070,
819 0x8cc, 0x000fffff, 0x00040035,
820 0x3f90, 0xffff0000, 0xfff00000,
821 0x9148, 0xffff0000, 0xfff00000,
822 0x3f94, 0xffff0000, 0xfff00000,
823 0x914c, 0xffff0000, 0xfff00000,
824 0xc78, 0x00000080, 0x00000080,
825 0xbd4, 0x00073007, 0x00010002,
826 0xd02c, 0xbfffff1f, 0x08421000,
827 0xd0b8, 0x03773777, 0x02010002,
828 0x5bc0, 0x00200000, 0x50100000,
829 0x98f8, 0x33773777, 0x00010002,
830 0x98fc, 0xffffffff, 0x33221100,
831 0x7030, 0x31000311, 0x00000011,
832 0x2f48, 0x33773777, 0x00010002,
833 0x6b28, 0x00000010, 0x00000012,
834 0x7728, 0x00000010, 0x00000012,
835 0x10328, 0x00000010, 0x00000012,
836 0x10f28, 0x00000010, 0x00000012,
837 0x11b28, 0x00000010, 0x00000012,
838 0x12728, 0x00000010, 0x00000012,
839 0x240c, 0x000007ff, 0x00000380,
840 0x8a14, 0xf000001f, 0x00000007,
841 0x8b24, 0x3fff3fff, 0x00ff0fff,
842 0x8b10, 0x0000ff0f, 0x00000000,
843 0x28a4c, 0x07ffffff, 0x06000000,
844 0x10c, 0x00000001, 0x00010003,
845 0xa02c, 0xffffffff, 0x0000009b,
846 0x913c, 0x0000000f, 0x0100000a,
847 0x8d00, 0xffff7f7f, 0x100e4848,
848 0x8d04, 0x00ffffff, 0x00164745,
849 0x8c00, 0xfffc0003, 0xe4000003,
850 0x8c04, 0xf8ff00ff, 0x40600060,
851 0x8c08, 0x00ff00ff, 0x001c001c,
852 0x8cf0, 0x1fff1fff, 0x08e00410,
853 0x8c20, 0x0fff0fff, 0x00800080,
854 0x8c24, 0x0fff0fff, 0x00800080,
855 0x8c18, 0xffffffff, 0x20202078,
856 0x8c1c, 0x0000ffff, 0x00001010,
857 0x28350, 0x00000f01, 0x00000000,
858 0x9508, 0x3700001f, 0x00000002,
859 0x960c, 0xffffffff, 0x54763210,
860 0x88c4, 0x001f3ae3, 0x000000c2,
861 0x88d4, 0x0000001f, 0x00000010,
862 0x8974, 0xffffffff, 0x00000000
863};
864
865static const u32 caicos_golden_registers[] =
866{
867 0x5eb4, 0xffffffff, 0x00000002,
868 0x5e78, 0x8f311ff1, 0x001000f0,
869 0x8c8, 0x00003420, 0x00001450,
870 0x8cc, 0x000fffff, 0x00040035,
871 0x3f90, 0xffff0000, 0xfffc0000,
872 0x9148, 0xffff0000, 0xfffc0000,
873 0x3f94, 0xffff0000, 0xfffc0000,
874 0x914c, 0xffff0000, 0xfffc0000,
875 0xc78, 0x00000080, 0x00000080,
876 0xbd4, 0x00073007, 0x00010001,
877 0xd02c, 0xbfffff1f, 0x08421000,
878 0xd0b8, 0x03773777, 0x02010001,
879 0x5bc0, 0x00200000, 0x50100000,
880 0x98f8, 0x33773777, 0x02010001,
881 0x98fc, 0xffffffff, 0x33221100,
882 0x7030, 0x31000311, 0x00000011,
883 0x2f48, 0x33773777, 0x02010001,
884 0x6b28, 0x00000010, 0x00000012,
885 0x7728, 0x00000010, 0x00000012,
886 0x10328, 0x00000010, 0x00000012,
887 0x10f28, 0x00000010, 0x00000012,
888 0x11b28, 0x00000010, 0x00000012,
889 0x12728, 0x00000010, 0x00000012,
890 0x240c, 0x000007ff, 0x00000380,
891 0x8a14, 0xf000001f, 0x00000001,
892 0x8b24, 0x3fff3fff, 0x00ff0fff,
893 0x8b10, 0x0000ff0f, 0x00000000,
894 0x28a4c, 0x07ffffff, 0x06000000,
895 0x10c, 0x00000001, 0x00010003,
896 0xa02c, 0xffffffff, 0x0000009b,
897 0x913c, 0x0000000f, 0x0100000a,
898 0x8d00, 0xffff7f7f, 0x100e4848,
899 0x8d04, 0x00ffffff, 0x00164745,
900 0x8c00, 0xfffc0003, 0xe4000003,
901 0x8c04, 0xf8ff00ff, 0x40600060,
902 0x8c08, 0x00ff00ff, 0x001c001c,
903 0x8cf0, 0x1fff1fff, 0x08e00410,
904 0x8c20, 0x0fff0fff, 0x00800080,
905 0x8c24, 0x0fff0fff, 0x00800080,
906 0x8c18, 0xffffffff, 0x20202078,
907 0x8c1c, 0x0000ffff, 0x00001010,
908 0x28350, 0x00000f01, 0x00000000,
909 0x9508, 0x3700001f, 0x00000002,
910 0x960c, 0xffffffff, 0x54763210,
911 0x88c4, 0x001f3ae3, 0x000000c2,
912 0x88d4, 0x0000001f, 0x00000010,
913 0x8974, 0xffffffff, 0x00000000
914};
915
916static void evergreen_init_golden_registers(struct radeon_device *rdev)
917{
918 switch (rdev->family) {
919 case CHIP_CYPRESS:
920 case CHIP_HEMLOCK:
921 radeon_program_register_sequence(rdev,
922 evergreen_golden_registers,
923 (const u32)ARRAY_SIZE(evergreen_golden_registers));
924 radeon_program_register_sequence(rdev,
925 evergreen_golden_registers2,
926 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
927 radeon_program_register_sequence(rdev,
928 cypress_mgcg_init,
929 (const u32)ARRAY_SIZE(cypress_mgcg_init));
930 break;
931 case CHIP_JUNIPER:
932 radeon_program_register_sequence(rdev,
933 evergreen_golden_registers,
934 (const u32)ARRAY_SIZE(evergreen_golden_registers));
935 radeon_program_register_sequence(rdev,
936 evergreen_golden_registers2,
937 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
938 radeon_program_register_sequence(rdev,
939 juniper_mgcg_init,
940 (const u32)ARRAY_SIZE(juniper_mgcg_init));
941 break;
942 case CHIP_REDWOOD:
943 radeon_program_register_sequence(rdev,
944 evergreen_golden_registers,
945 (const u32)ARRAY_SIZE(evergreen_golden_registers));
946 radeon_program_register_sequence(rdev,
947 evergreen_golden_registers2,
948 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
949 radeon_program_register_sequence(rdev,
950 redwood_mgcg_init,
951 (const u32)ARRAY_SIZE(redwood_mgcg_init));
952 break;
953 case CHIP_CEDAR:
954 radeon_program_register_sequence(rdev,
955 cedar_golden_registers,
956 (const u32)ARRAY_SIZE(cedar_golden_registers));
957 radeon_program_register_sequence(rdev,
958 evergreen_golden_registers2,
959 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
960 radeon_program_register_sequence(rdev,
961 cedar_mgcg_init,
962 (const u32)ARRAY_SIZE(cedar_mgcg_init));
963 break;
964 case CHIP_PALM:
965 radeon_program_register_sequence(rdev,
966 wrestler_golden_registers,
967 (const u32)ARRAY_SIZE(wrestler_golden_registers));
968 break;
969 case CHIP_SUMO:
970 radeon_program_register_sequence(rdev,
971 supersumo_golden_registers,
972 (const u32)ARRAY_SIZE(supersumo_golden_registers));
973 break;
974 case CHIP_SUMO2:
975 radeon_program_register_sequence(rdev,
976 supersumo_golden_registers,
977 (const u32)ARRAY_SIZE(supersumo_golden_registers));
978 radeon_program_register_sequence(rdev,
979 sumo_golden_registers,
980 (const u32)ARRAY_SIZE(sumo_golden_registers));
981 break;
982 case CHIP_BARTS:
983 radeon_program_register_sequence(rdev,
984 barts_golden_registers,
985 (const u32)ARRAY_SIZE(barts_golden_registers));
986 break;
987 case CHIP_TURKS:
988 radeon_program_register_sequence(rdev,
989 turks_golden_registers,
990 (const u32)ARRAY_SIZE(turks_golden_registers));
991 break;
992 case CHIP_CAICOS:
993 radeon_program_register_sequence(rdev,
994 caicos_golden_registers,
995 (const u32)ARRAY_SIZE(caicos_golden_registers));
996 break;
997 default:
998 break;
999 }
1000}
1001
285484e2
JG		 1002void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1003 unsigned *bankh, unsigned *mtaspect,
1004 unsigned *tile_split)
1005{
1006 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1007 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1008 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1009 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1010 switch (*bankw) {
1011 default:
1012 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1013 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1014 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1015 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1016 }
1017 switch (*bankh) {
1018 default:
1019 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1020 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1021 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1022 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1023 }
1024 switch (*mtaspect) {
1025 default:
1026 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1027 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1028 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1029 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1030 }
1031}
1032
23d33ba3
AD		 1033static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1034 u32 cntl_reg, u32 status_reg)
1035{
1036 int r, i;
1037 struct atom_clock_dividers dividers;
1038
1039 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1040 clock, false, &dividers);
1041 if (r)
1042 return r;
1043
1044 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1045
1046 for (i = 0; i < 100; i++) {
1047 if (RREG32(status_reg) & DCLK_STATUS)
1048 break;
1049 mdelay(10);
1050 }
1051 if (i == 100)
1052 return -ETIMEDOUT;
1053
1054 return 0;
1055}
1056
1057int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1058{
1059 int r = 0;
1060 u32 cg_scratch = RREG32(CG_SCRATCH1);
1061
1062 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1063 if (r)
1064 goto done;
1065 cg_scratch &= 0xffff0000;
1066 cg_scratch |= vclk / 100; /* Mhz */
1067
1068 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1069 if (r)
1070 goto done;
1071 cg_scratch &= 0x0000ffff;
1072 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1073
1074done:
1075 WREG32(CG_SCRATCH1, cg_scratch);
1076
1077 return r;
1078}
1079
a8b4925c
AD		 1080int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1081{
1082 /* start off with something large */
facd112d 1083 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1084 int r;
1085
4ed10835
CK		 1086 /* bypass vclk and dclk with bclk */
1087 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1088 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1089 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1090
1091 /* put PLL in bypass mode */
1092 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1093
1094 if (!vclk || !dclk) {
1095 /* keep the Bypass mode, put PLL to sleep */
1096 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1097 return 0;
1098 }
1099
facd112d
CK		 1100 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1101 16384, 0x03FFFFFF, 0, 128, 5,
1102 &fb_div, &vclk_div, &dclk_div);
1103 if (r)
1104 return r;
a8b4925c
AD		 1105
1106 /* set VCO_MODE to 1 */
1107 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1108
1109 /* toggle UPLL_SLEEP to 1 then back to 0 */
1110 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1111 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1112
1113 /* deassert UPLL_RESET */
1114 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1115
1116 mdelay(1);
1117
facd112d 1118 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1119 if (r)
1120 return r;
1121
1122 /* assert UPLL_RESET again */
1123 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1124
1125 /* disable spread spectrum. */
1126 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1127
1128 /* set feedback divider */
facd112d 1129 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1130
1131 /* set ref divider to 0 */
1132 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1133
facd112d 1134 if (fb_div < 307200)
a8b4925c
AD		 1135 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1136 else
1137 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1138
1139 /* set PDIV_A and PDIV_B */
1140 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1141 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1142 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1143
1144 /* give the PLL some time to settle */
1145 mdelay(15);
1146
1147 /* deassert PLL_RESET */
1148 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1149
1150 mdelay(15);
1151
1152 /* switch from bypass mode to normal mode */
1153 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1154
facd112d 1155 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1156 if (r)
1157 return r;
1158
1159 /* switch VCLK and DCLK selection */
1160 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1161 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1162 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1163
1164 mdelay(100);
1165
1166 return 0;
1167}
1168
d054ac16
AD		 1169void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1170{
1171 u16 ctl, v;
32195aec 1172 int err;
d054ac16 1173
32195aec 1174 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
d054ac16
AD		 1175 if (err)
1176 return;
1177
1178 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1179
1180 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1181 * to avoid hangs or perfomance issues
1182 */
1183 if ((v == 0) || (v == 6) || (v == 7)) {
1184 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1185 ctl |= (2 << 12);
32195aec 1186 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
d054ac16
AD		 1187 }
1188}
1189
10257a6d
AD		 1190static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1191{
1192 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1193 return true;
1194 else
1195 return false;
1196}
1197
1198static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1199{
1200 u32 pos1, pos2;
1201
1202 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1203 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1204
1205 if (pos1 != pos2)
1206 return true;
1207 else
1208 return false;
1209}
1210
377edc8b
AD		 1211/**
1212 * dce4_wait_for_vblank - vblank wait asic callback.
1213 *
1214 * @rdev: radeon_device pointer
1215 * @crtc: crtc to wait for vblank on
1216 *
1217 * Wait for vblank on the requested crtc (evergreen+).
1218 */
3ae19b75
AD		 1219void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1220{
10257a6d 1221 unsigned i = 0;
3ae19b75 1222
4a15903d
AD		 1223 if (crtc >= rdev->num_crtc)
1224 return;
1225
10257a6d
AD		 1226 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1227 return;
1228
1229 /* depending on when we hit vblank, we may be close to active; if so,
1230 * wait for another frame.
1231 */
1232 while (dce4_is_in_vblank(rdev, crtc)) {
1233 if (i++ % 100 == 0) {
1234 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1235 break;
3ae19b75 1236 }
10257a6d
AD		 1237 }
1238
1239 while (!dce4_is_in_vblank(rdev, crtc)) {
1240 if (i++ % 100 == 0) {
1241 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1242 break;
3ae19b75
AD		 1243 }
1244 }
1245}
1246
377edc8b
AD		 1247/**
1248 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1249 *
1250 * @rdev: radeon_device pointer
1251 * @crtc: crtc to prepare for pageflip on
1252 *
1253 * Pre-pageflip callback (evergreen+).
1254 * Enables the pageflip irq (vblank irq).
1255 */
6f34be50
AD		 1256void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1257{
6f34be50
AD		 1258 /* enable the pflip int */
1259 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1260}
1261
377edc8b
AD		 1262/**
1263 * evergreen_post_page_flip - pos-pageflip callback.
1264 *
1265 * @rdev: radeon_device pointer
1266 * @crtc: crtc to cleanup pageflip on
1267 *
1268 * Post-pageflip callback (evergreen+).
1269 * Disables the pageflip irq (vblank irq).
1270 */
6f34be50
AD		 1271void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1272{
1273 /* disable the pflip int */
1274 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1275}
1276
377edc8b
AD		 1277/**
1278 * evergreen_page_flip - pageflip callback.
1279 *
1280 * @rdev: radeon_device pointer
1281 * @crtc_id: crtc to cleanup pageflip on
1282 * @crtc_base: new address of the crtc (GPU MC address)
1283 *
1284 * Does the actual pageflip (evergreen+).
1285 * During vblank we take the crtc lock and wait for the update_pending
1286 * bit to go high, when it does, we release the lock, and allow the
1287 * double buffered update to take place.
1288 * Returns the current update pending status.
1289 */
6f34be50
AD		 1290u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1291{
1292 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1293 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1294 int i;
6f34be50
AD		 1295
1296 /* Lock the graphics update lock */
1297 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1298 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1299
1300 /* update the scanout addresses */
1301 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1302 upper_32_bits(crtc_base));
1303 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1304 (u32)crtc_base);
1305
1306 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1307 upper_32_bits(crtc_base));
1308 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1309 (u32)crtc_base);
1310
1311 /* Wait for update_pending to go high. */
f6496479
AD		 1312 for (i = 0; i < rdev->usec_timeout; i++) {
1313 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1314 break;
1315 udelay(1);
1316 }
6f34be50
AD		 1317 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1318
1319 /* Unlock the lock, so double-buffering can take place inside vblank */
1320 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1321 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1322
1323 /* Return current update_pending status: */
1324 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1325}
1326
21a8122a 1327/* get temperature in millidegrees */
20d391d7 1328int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1329{
1c88d74f
AD		 1330 u32 temp, toffset;
1331 int actual_temp = 0;
67b3f823
AD		 1332
1333 if (rdev->family == CHIP_JUNIPER) {
1334 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1335 TOFFSET_SHIFT;
1336 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1337 TS0_ADC_DOUT_SHIFT;
1338
1339 if (toffset & 0x100)
1340 actual_temp = temp / 2 - (0x200 - toffset);
1341 else
1342 actual_temp = temp / 2 + toffset;
1343
1344 actual_temp = actual_temp * 1000;
1345
1346 } else {
1347 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1348 ASIC_T_SHIFT;
1349
1350 if (temp & 0x400)
1351 actual_temp = -256;
1352 else if (temp & 0x200)
1353 actual_temp = 255;
1354 else if (temp & 0x100) {
1355 actual_temp = temp & 0x1ff;
1356 actual_temp |= ~0x1ff;
1357 } else
1358 actual_temp = temp & 0xff;
1359
1360 actual_temp = (actual_temp * 1000) / 2;
1361 }
21a8122a 1362
67b3f823 1363 return actual_temp;
21a8122a
AD		 1364}
1365
20d391d7 1366int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1367{
1368 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1369 int actual_temp = temp - 49;
e33df25f
AD		 1370
1371 return actual_temp * 1000;
1372}
1373
377edc8b
AD		 1374/**
1375 * sumo_pm_init_profile - Initialize power profiles callback.
1376 *
1377 * @rdev: radeon_device pointer
1378 *
1379 * Initialize the power states used in profile mode
1380 * (sumo, trinity, SI).
1381 * Used for profile mode only.
1382 */
a4c9e2ee
AD		 1383void sumo_pm_init_profile(struct radeon_device *rdev)
1384{
1385 int idx;
1386
1387 /* default */
1388 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1389 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1390 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1391 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1392
1393 /* low,mid sh/mh */
1394 if (rdev->flags & RADEON_IS_MOBILITY)
1395 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1396 else
1397 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1398
1399 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1400 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1401 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1402 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1403
1404 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1405 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1406 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1407 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1408
1409 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1410 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1411 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1412 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1413
1414 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1416 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1417 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1418
1419 /* high sh/mh */
1420 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1421 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1422 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1423 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1424 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1425 rdev->pm.power_state[idx].num_clock_modes - 1;
1426
1427 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1428 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1429 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1430 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1431 rdev->pm.power_state[idx].num_clock_modes - 1;
1432}
1433
27810fb2
AD		 1434/**
1435 * btc_pm_init_profile - Initialize power profiles callback.
1436 *
1437 * @rdev: radeon_device pointer
1438 *
1439 * Initialize the power states used in profile mode
1440 * (BTC, cayman).
1441 * Used for profile mode only.
1442 */
1443void btc_pm_init_profile(struct radeon_device *rdev)
1444{
1445 int idx;
1446
1447 /* default */
1448 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1450 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1451 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1452 /* starting with BTC, there is one state that is used for both
1453 * MH and SH. Difference is that we always use the high clock index for
1454 * mclk.
1455 */
1456 if (rdev->flags & RADEON_IS_MOBILITY)
1457 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1458 else
1459 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1460 /* low sh */
1461 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1462 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1463 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1464 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1465 /* mid sh */
1466 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1469 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1470 /* high sh */
1471 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1472 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1473 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1474 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1475 /* low mh */
1476 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1477 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1478 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1479 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1480 /* mid mh */
1481 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1483 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1484 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1485 /* high mh */
1486 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1488 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1489 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1490}
1491
377edc8b
AD		 1492/**
1493 * evergreen_pm_misc - set additional pm hw parameters callback.
1494 *
1495 * @rdev: radeon_device pointer
1496 *
1497 * Set non-clock parameters associated with a power state
1498 * (voltage, etc.) (evergreen+).
1499 */
49e02b73
AD		 1500void evergreen_pm_misc(struct radeon_device *rdev)
1501{
a081a9d6
RM		 1502 int req_ps_idx = rdev->pm.requested_power_state_index;
1503 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1504 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1505 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1506
2feea49a 1507 if (voltage->type == VOLTAGE_SW) {
c6cf7777
AD		 1508 /* 0xff0x are flags rather then an actual voltage */
1509 if ((voltage->voltage & 0xff00) == 0xff00)
a377e187 1510 return;
2feea49a 1511 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1512 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1513 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1514 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1515 }
7ae764b1
AD		 1516
1517 /* starting with BTC, there is one state that is used for both
1518 * MH and SH. Difference is that we always use the high clock index for
1519 * mclk and vddci.
1520 */
1521 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1522 (rdev->family >= CHIP_BARTS) &&
1523 rdev->pm.active_crtc_count &&
1524 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1525 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1526 voltage = &rdev->pm.power_state[req_ps_idx].
1527 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1528
c6cf7777
AD		 1529 /* 0xff0x are flags rather then an actual voltage */
1530 if ((voltage->vddci & 0xff00) == 0xff00)
a377e187 1531 return;
2feea49a
AD		 1532 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1533 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1534 rdev->pm.current_vddci = voltage->vddci;
1535 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1536 }
1537 }
49e02b73
AD		 1538}
1539
377edc8b
AD		 1540/**
1541 * evergreen_pm_prepare - pre-power state change callback.
1542 *
1543 * @rdev: radeon_device pointer
1544 *
1545 * Prepare for a power state change (evergreen+).
1546 */
49e02b73
AD		 1547void evergreen_pm_prepare(struct radeon_device *rdev)
1548{
1549 struct drm_device *ddev = rdev->ddev;
1550 struct drm_crtc *crtc;
1551 struct radeon_crtc *radeon_crtc;
1552 u32 tmp;
1553
1554 /* disable any active CRTCs */
1555 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1556 radeon_crtc = to_radeon_crtc(crtc);
1557 if (radeon_crtc->enabled) {
1558 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1559 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1560 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1561 }
1562 }
1563}
1564
377edc8b
AD		 1565/**
1566 * evergreen_pm_finish - post-power state change callback.
1567 *
1568 * @rdev: radeon_device pointer
1569 *
1570 * Clean up after a power state change (evergreen+).
1571 */
49e02b73
AD		 1572void evergreen_pm_finish(struct radeon_device *rdev)
1573{
1574 struct drm_device *ddev = rdev->ddev;
1575 struct drm_crtc *crtc;
1576 struct radeon_crtc *radeon_crtc;
1577 u32 tmp;
1578
1579 /* enable any active CRTCs */
1580 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1581 radeon_crtc = to_radeon_crtc(crtc);
1582 if (radeon_crtc->enabled) {
1583 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1584 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1585 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1586 }
1587 }
1588}
1589
377edc8b
AD		 1590/**
1591 * evergreen_hpd_sense - hpd sense callback.
1592 *
1593 * @rdev: radeon_device pointer
1594 * @hpd: hpd (hotplug detect) pin
1595 *
1596 * Checks if a digital monitor is connected (evergreen+).
1597 * Returns true if connected, false if not connected.
1598 */
bcc1c2a1
AD		 1599bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1600{
1601 bool connected = false;
0ca2ab52
AD		 1602
1603 switch (hpd) {
1604 case RADEON_HPD_1:
1605 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1606 connected = true;
1607 break;
1608 case RADEON_HPD_2:
1609 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1610 connected = true;
1611 break;
1612 case RADEON_HPD_3:
1613 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1614 connected = true;
1615 break;
1616 case RADEON_HPD_4:
1617 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1618 connected = true;
1619 break;
1620 case RADEON_HPD_5:
1621 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1622 connected = true;
1623 break;
1624 case RADEON_HPD_6:
1625 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1626 connected = true;
1627 break;
1628 default:
1629 break;
1630 }
1631
bcc1c2a1
AD		 1632 return connected;
1633}
1634
377edc8b
AD		 1635/**
1636 * evergreen_hpd_set_polarity - hpd set polarity callback.
1637 *
1638 * @rdev: radeon_device pointer
1639 * @hpd: hpd (hotplug detect) pin
1640 *
1641 * Set the polarity of the hpd pin (evergreen+).
1642 */
bcc1c2a1
AD		 1643void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1644 enum radeon_hpd_id hpd)
1645{
0ca2ab52
AD		 1646 u32 tmp;
1647 bool connected = evergreen_hpd_sense(rdev, hpd);
1648
1649 switch (hpd) {
1650 case RADEON_HPD_1:
1651 tmp = RREG32(DC_HPD1_INT_CONTROL);
1652 if (connected)
1653 tmp &= ~DC_HPDx_INT_POLARITY;
1654 else
1655 tmp |= DC_HPDx_INT_POLARITY;
1656 WREG32(DC_HPD1_INT_CONTROL, tmp);
1657 break;
1658 case RADEON_HPD_2:
1659 tmp = RREG32(DC_HPD2_INT_CONTROL);
1660 if (connected)
1661 tmp &= ~DC_HPDx_INT_POLARITY;
1662 else
1663 tmp |= DC_HPDx_INT_POLARITY;
1664 WREG32(DC_HPD2_INT_CONTROL, tmp);
1665 break;
1666 case RADEON_HPD_3:
1667 tmp = RREG32(DC_HPD3_INT_CONTROL);
1668 if (connected)
1669 tmp &= ~DC_HPDx_INT_POLARITY;
1670 else
1671 tmp |= DC_HPDx_INT_POLARITY;
1672 WREG32(DC_HPD3_INT_CONTROL, tmp);
1673 break;
1674 case RADEON_HPD_4:
1675 tmp = RREG32(DC_HPD4_INT_CONTROL);
1676 if (connected)
1677 tmp &= ~DC_HPDx_INT_POLARITY;
1678 else
1679 tmp |= DC_HPDx_INT_POLARITY;
1680 WREG32(DC_HPD4_INT_CONTROL, tmp);
1681 break;
1682 case RADEON_HPD_5:
1683 tmp = RREG32(DC_HPD5_INT_CONTROL);
1684 if (connected)
1685 tmp &= ~DC_HPDx_INT_POLARITY;
1686 else
1687 tmp |= DC_HPDx_INT_POLARITY;
1688 WREG32(DC_HPD5_INT_CONTROL, tmp);
1689 break;
1690 case RADEON_HPD_6:
1691 tmp = RREG32(DC_HPD6_INT_CONTROL);
1692 if (connected)
1693 tmp &= ~DC_HPDx_INT_POLARITY;
1694 else
1695 tmp |= DC_HPDx_INT_POLARITY;
1696 WREG32(DC_HPD6_INT_CONTROL, tmp);
1697 break;
1698 default:
1699 break;
1700 }
bcc1c2a1
AD		 1701}
1702
377edc8b
AD		 1703/**
1704 * evergreen_hpd_init - hpd setup callback.
1705 *
1706 * @rdev: radeon_device pointer
1707 *
1708 * Setup the hpd pins used by the card (evergreen+).
1709 * Enable the pin, set the polarity, and enable the hpd interrupts.
1710 */
bcc1c2a1
AD		 1711void evergreen_hpd_init(struct radeon_device *rdev)
1712{
0ca2ab52
AD		 1713 struct drm_device *dev = rdev->ddev;
1714 struct drm_connector *connector;
fb98257a 1715 unsigned enabled = 0;
0ca2ab52
AD		 1716 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1717 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1718
0ca2ab52
AD		 1719 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1720 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1721
1722 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1723 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1724 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1725 * aux dp channel on imac and help (but not completely fix)
1726 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1727 * also avoid interrupt storms during dpms.
1728 */
1729 continue;
1730 }
0ca2ab52
AD		 1731 switch (radeon_connector->hpd.hpd) {
1732 case RADEON_HPD_1:
1733 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1734 break;
1735 case RADEON_HPD_2:
1736 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1737 break;
1738 case RADEON_HPD_3:
1739 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1740 break;
1741 case RADEON_HPD_4:
1742 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1743 break;
1744 case RADEON_HPD_5:
1745 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1746 break;
1747 case RADEON_HPD_6:
1748 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1749 break;
1750 default:
1751 break;
1752 }
64912e99 1753 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1754 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1755 }
fb98257a 1756 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1757}
1758
377edc8b
AD		 1759/**
1760 * evergreen_hpd_fini - hpd tear down callback.
1761 *
1762 * @rdev: radeon_device pointer
1763 *
1764 * Tear down the hpd pins used by the card (evergreen+).
1765 * Disable the hpd interrupts.
1766 */
0ca2ab52 1767void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1768{
0ca2ab52
AD		 1769 struct drm_device *dev = rdev->ddev;
1770 struct drm_connector *connector;
fb98257a 1771 unsigned disabled = 0;
0ca2ab52
AD		 1772
1773 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1774 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1775 switch (radeon_connector->hpd.hpd) {
1776 case RADEON_HPD_1:
1777 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1778 break;
1779 case RADEON_HPD_2:
1780 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1781 break;
1782 case RADEON_HPD_3:
1783 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1784 break;
1785 case RADEON_HPD_4:
1786 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1787 break;
1788 case RADEON_HPD_5:
1789 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1790 break;
1791 case RADEON_HPD_6:
1792 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1793 break;
1794 default:
1795 break;
1796 }
fb98257a 1797 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1798 }
fb98257a 1799 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1800}
1801
f9d9c362
AD		 1802/* watermark setup */
1803
1804static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1805 struct radeon_crtc *radeon_crtc,
1806 struct drm_display_mode *mode,
1807 struct drm_display_mode *other_mode)
1808{
12dfc843 1809 u32 tmp;
f9d9c362
AD		 1810 /*
1811 * Line Buffer Setup
1812 * There are 3 line buffers, each one shared by 2 display controllers.
1813 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1814 * the display controllers. The paritioning is done via one of four
1815 * preset allocations specified in bits 2:0:
1816 * first display controller
1817 * 0 - first half of lb (3840 * 2)
1818 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1819 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1820 * 3 - first 1/4 of lb (1920 * 2)
1821 * second display controller
1822 * 4 - second half of lb (3840 * 2)
1823 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1824 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1825 * 7 - last 1/4 of lb (1920 * 2)
1826 */
12dfc843
AD		 1827 /* this can get tricky if we have two large displays on a paired group
1828 * of crtcs. Ideally for multiple large displays we'd assign them to
1829 * non-linked crtcs for maximum line buffer allocation.
1830 */
1831 if (radeon_crtc->base.enabled && mode) {
1832 if (other_mode)
f9d9c362 1833 tmp = 0; /* 1/2 */
12dfc843
AD		 1834 else
1835 tmp = 2; /* whole */
1836 } else
1837 tmp = 0;
f9d9c362
AD		 1838
1839 /* second controller of the pair uses second half of the lb */
1840 if (radeon_crtc->crtc_id % 2)
1841 tmp += 4;
1842 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1843
12dfc843
AD		 1844 if (radeon_crtc->base.enabled && mode) {
1845 switch (tmp) {
1846 case 0:
1847 case 4:
1848 default:
1849 if (ASIC_IS_DCE5(rdev))
1850 return 4096 * 2;
1851 else
1852 return 3840 * 2;
1853 case 1:
1854 case 5:
1855 if (ASIC_IS_DCE5(rdev))
1856 return 6144 * 2;
1857 else
1858 return 5760 * 2;
1859 case 2:
1860 case 6:
1861 if (ASIC_IS_DCE5(rdev))
1862 return 8192 * 2;
1863 else
1864 return 7680 * 2;
1865 case 3:
1866 case 7:
1867 if (ASIC_IS_DCE5(rdev))
1868 return 2048 * 2;
1869 else
1870 return 1920 * 2;
1871 }
f9d9c362 1872 }
12dfc843
AD		 1873
1874 /* controller not enabled, so no lb used */
1875 return 0;
f9d9c362
AD		 1876}
1877
ca7db22b 1878u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1879{
1880 u32 tmp = RREG32(MC_SHARED_CHMAP);
1881
1882 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1883 case 0:
1884 default:
1885 return 1;
1886 case 1:
1887 return 2;
1888 case 2:
1889 return 4;
1890 case 3:
1891 return 8;
1892 }
1893}
1894
1895struct evergreen_wm_params {
1896 u32 dram_channels; /* number of dram channels */
1897 u32 yclk; /* bandwidth per dram data pin in kHz */
1898 u32 sclk; /* engine clock in kHz */
1899 u32 disp_clk; /* display clock in kHz */
1900 u32 src_width; /* viewport width */
1901 u32 active_time; /* active display time in ns */
1902 u32 blank_time; /* blank time in ns */
1903 bool interlaced; /* mode is interlaced */
1904 fixed20_12 vsc; /* vertical scale ratio */
1905 u32 num_heads; /* number of active crtcs */
1906 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1907 u32 lb_size; /* line buffer allocated to pipe */
1908 u32 vtaps; /* vertical scaler taps */
1909};
1910
1911static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1912{
1913 /* Calculate DRAM Bandwidth and the part allocated to display. */
1914 fixed20_12 dram_efficiency; /* 0.7 */
1915 fixed20_12 yclk, dram_channels, bandwidth;
1916 fixed20_12 a;
1917
1918 a.full = dfixed_const(1000);
1919 yclk.full = dfixed_const(wm->yclk);
1920 yclk.full = dfixed_div(yclk, a);
1921 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1922 a.full = dfixed_const(10);
1923 dram_efficiency.full = dfixed_const(7);
1924 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1925 bandwidth.full = dfixed_mul(dram_channels, yclk);
1926 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1927
1928 return dfixed_trunc(bandwidth);
1929}
1930
1931static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1932{
1933 /* Calculate DRAM Bandwidth and the part allocated to display. */
1934 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1935 fixed20_12 yclk, dram_channels, bandwidth;
1936 fixed20_12 a;
1937
1938 a.full = dfixed_const(1000);
1939 yclk.full = dfixed_const(wm->yclk);
1940 yclk.full = dfixed_div(yclk, a);
1941 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1942 a.full = dfixed_const(10);
1943 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1944 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1945 bandwidth.full = dfixed_mul(dram_channels, yclk);
1946 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1947
1948 return dfixed_trunc(bandwidth);
1949}
1950
1951static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1952{
1953 /* Calculate the display Data return Bandwidth */
1954 fixed20_12 return_efficiency; /* 0.8 */
1955 fixed20_12 sclk, bandwidth;
1956 fixed20_12 a;
1957
1958 a.full = dfixed_const(1000);
1959 sclk.full = dfixed_const(wm->sclk);
1960 sclk.full = dfixed_div(sclk, a);
1961 a.full = dfixed_const(10);
1962 return_efficiency.full = dfixed_const(8);
1963 return_efficiency.full = dfixed_div(return_efficiency, a);
1964 a.full = dfixed_const(32);
1965 bandwidth.full = dfixed_mul(a, sclk);
1966 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1967
1968 return dfixed_trunc(bandwidth);
1969}
1970
1971static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1972{
1973 /* Calculate the DMIF Request Bandwidth */
1974 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1975 fixed20_12 disp_clk, bandwidth;
1976 fixed20_12 a;
1977
1978 a.full = dfixed_const(1000);
1979 disp_clk.full = dfixed_const(wm->disp_clk);
1980 disp_clk.full = dfixed_div(disp_clk, a);
1981 a.full = dfixed_const(10);
1982 disp_clk_request_efficiency.full = dfixed_const(8);
1983 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1984 a.full = dfixed_const(32);
1985 bandwidth.full = dfixed_mul(a, disp_clk);
1986 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1987
1988 return dfixed_trunc(bandwidth);
1989}
1990
1991static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1992{
1993 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1994 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1995 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1996 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1997
1998 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1999}
2000
2001static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2002{
2003 /* Calculate the display mode Average Bandwidth
2004 * DisplayMode should contain the source and destination dimensions,
2005 * timing, etc.
2006 */
2007 fixed20_12 bpp;
2008 fixed20_12 line_time;
2009 fixed20_12 src_width;
2010 fixed20_12 bandwidth;
2011 fixed20_12 a;
2012
2013 a.full = dfixed_const(1000);
2014 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2015 line_time.full = dfixed_div(line_time, a);
2016 bpp.full = dfixed_const(wm->bytes_per_pixel);
2017 src_width.full = dfixed_const(wm->src_width);
2018 bandwidth.full = dfixed_mul(src_width, bpp);
2019 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2020 bandwidth.full = dfixed_div(bandwidth, line_time);
2021
2022 return dfixed_trunc(bandwidth);
2023}
2024
2025static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2026{
2027 /* First calcualte the latency in ns */
2028 u32 mc_latency = 2000; /* 2000 ns. */
2029 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2030 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2031 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2032 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2033 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2034 (wm->num_heads * cursor_line_pair_return_time);
2035 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2036 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2037 fixed20_12 a, b, c;
2038
2039 if (wm->num_heads == 0)
2040 return 0;
2041
2042 a.full = dfixed_const(2);
2043 b.full = dfixed_const(1);
2044 if ((wm->vsc.full > a.full) ||
2045 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2046 (wm->vtaps >= 5) ||
2047 ((wm->vsc.full >= a.full) && wm->interlaced))
2048 max_src_lines_per_dst_line = 4;
2049 else
2050 max_src_lines_per_dst_line = 2;
2051
2052 a.full = dfixed_const(available_bandwidth);
2053 b.full = dfixed_const(wm->num_heads);
2054 a.full = dfixed_div(a, b);
2055
2056 b.full = dfixed_const(1000);
2057 c.full = dfixed_const(wm->disp_clk);
2058 b.full = dfixed_div(c, b);
2059 c.full = dfixed_const(wm->bytes_per_pixel);
2060 b.full = dfixed_mul(b, c);
2061
2062 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2063
2064 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2065 b.full = dfixed_const(1000);
2066 c.full = dfixed_const(lb_fill_bw);
2067 b.full = dfixed_div(c, b);
2068 a.full = dfixed_div(a, b);
2069 line_fill_time = dfixed_trunc(a);
2070
2071 if (line_fill_time < wm->active_time)
2072 return latency;
2073 else
2074 return latency + (line_fill_time - wm->active_time);
2075
2076}
2077
2078static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2079{
2080 if (evergreen_average_bandwidth(wm) <=
2081 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2082 return true;
2083 else
2084 return false;
2085};
2086
2087static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2088{
2089 if (evergreen_average_bandwidth(wm) <=
2090 (evergreen_available_bandwidth(wm) / wm->num_heads))
2091 return true;
2092 else
2093 return false;
2094};
2095
2096static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2097{
2098 u32 lb_partitions = wm->lb_size / wm->src_width;
2099 u32 line_time = wm->active_time + wm->blank_time;
2100 u32 latency_tolerant_lines;
2101 u32 latency_hiding;
2102 fixed20_12 a;
2103
2104 a.full = dfixed_const(1);
2105 if (wm->vsc.full > a.full)
2106 latency_tolerant_lines = 1;
2107 else {
2108 if (lb_partitions <= (wm->vtaps + 1))
2109 latency_tolerant_lines = 1;
2110 else
2111 latency_tolerant_lines = 2;
2112 }
2113
2114 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2115
2116 if (evergreen_latency_watermark(wm) <= latency_hiding)
2117 return true;
2118 else
2119 return false;
2120}
2121
2122static void evergreen_program_watermarks(struct radeon_device *rdev,
2123 struct radeon_crtc *radeon_crtc,
2124 u32 lb_size, u32 num_heads)
2125{
2126 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2127 struct evergreen_wm_params wm_low, wm_high;
2128 u32 dram_channels;
f9d9c362
AD		 2129 u32 pixel_period;
2130 u32 line_time = 0;
2131 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2132 u32 priority_a_mark = 0, priority_b_mark = 0;
2133 u32 priority_a_cnt = PRIORITY_OFF;
2134 u32 priority_b_cnt = PRIORITY_OFF;
2135 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2136 u32 tmp, arb_control3;
2137 fixed20_12 a, b, c;
2138
2139 if (radeon_crtc->base.enabled && num_heads && mode) {
2140 pixel_period = 1000000 / (u32)mode->clock;
2141 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2142 priority_a_cnt = 0;
2143 priority_b_cnt = 0;
cf0cfdd7
AD		 2144 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2145
2146 /* watermark for high clocks */
2147 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2148 wm_high.yclk =
2149 radeon_dpm_get_mclk(rdev, false) * 10;
2150 wm_high.sclk =
2151 radeon_dpm_get_sclk(rdev, false) * 10;
2152 } else {
2153 wm_high.yclk = rdev->pm.current_mclk * 10;
2154 wm_high.sclk = rdev->pm.current_sclk * 10;
2155 }
f9d9c362 2156
cf0cfdd7
AD		 2157 wm_high.disp_clk = mode->clock;
2158 wm_high.src_width = mode->crtc_hdisplay;
2159 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2160 wm_high.blank_time = line_time - wm_high.active_time;
2161 wm_high.interlaced = false;
f9d9c362 2162 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2163 wm_high.interlaced = true;
2164 wm_high.vsc = radeon_crtc->vsc;
2165 wm_high.vtaps = 1;
f9d9c362 2166 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2167 wm_high.vtaps = 2;
2168 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2169 wm_high.lb_size = lb_size;
2170 wm_high.dram_channels = dram_channels;
2171 wm_high.num_heads = num_heads;
2172
2173 /* watermark for low clocks */
2174 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2175 wm_low.yclk =
2176 radeon_dpm_get_mclk(rdev, true) * 10;
2177 wm_low.sclk =
2178 radeon_dpm_get_sclk(rdev, true) * 10;
2179 } else {
2180 wm_low.yclk = rdev->pm.current_mclk * 10;
2181 wm_low.sclk = rdev->pm.current_sclk * 10;
2182 }
2183
2184 wm_low.disp_clk = mode->clock;
2185 wm_low.src_width = mode->crtc_hdisplay;
2186 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2187 wm_low.blank_time = line_time - wm_low.active_time;
2188 wm_low.interlaced = false;
2189 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2190 wm_low.interlaced = true;
2191 wm_low.vsc = radeon_crtc->vsc;
2192 wm_low.vtaps = 1;
2193 if (radeon_crtc->rmx_type != RMX_OFF)
2194 wm_low.vtaps = 2;
2195 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2196 wm_low.lb_size = lb_size;
2197 wm_low.dram_channels = dram_channels;
2198 wm_low.num_heads = num_heads;
f9d9c362
AD		 2199
2200 /* set for high clocks */
cf0cfdd7 2201 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2202 /* set for low clocks */
cf0cfdd7 2203 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2204
2205 /* possibly force display priority to high */
2206 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2207 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2208 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2209 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2210 (rdev->disp_priority == 2)) {
cf0cfdd7 2211 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2212 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2213 }
2214 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2215 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2216 !evergreen_check_latency_hiding(&wm_low) ||
2217 (rdev->disp_priority == 2)) {
2218 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2219 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2220 }
2221
2222 a.full = dfixed_const(1000);
2223 b.full = dfixed_const(mode->clock);
2224 b.full = dfixed_div(b, a);
2225 c.full = dfixed_const(latency_watermark_a);
2226 c.full = dfixed_mul(c, b);
2227 c.full = dfixed_mul(c, radeon_crtc->hsc);
2228 c.full = dfixed_div(c, a);
2229 a.full = dfixed_const(16);
2230 c.full = dfixed_div(c, a);
2231 priority_a_mark = dfixed_trunc(c);
2232 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2233
2234 a.full = dfixed_const(1000);
2235 b.full = dfixed_const(mode->clock);
2236 b.full = dfixed_div(b, a);
2237 c.full = dfixed_const(latency_watermark_b);
2238 c.full = dfixed_mul(c, b);
2239 c.full = dfixed_mul(c, radeon_crtc->hsc);
2240 c.full = dfixed_div(c, a);
2241 a.full = dfixed_const(16);
2242 c.full = dfixed_div(c, a);
2243 priority_b_mark = dfixed_trunc(c);
2244 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2245 }
2246
2247 /* select wm A */
2248 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2249 tmp = arb_control3;
2250 tmp &= ~LATENCY_WATERMARK_MASK(3);
2251 tmp |= LATENCY_WATERMARK_MASK(1);
2252 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2253 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2254 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2255 LATENCY_HIGH_WATERMARK(line_time)));
2256 /* select wm B */
2257 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2258 tmp &= ~LATENCY_WATERMARK_MASK(3);
2259 tmp |= LATENCY_WATERMARK_MASK(2);
2260 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2261 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2262 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2263 LATENCY_HIGH_WATERMARK(line_time)));
2264 /* restore original selection */
2265 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2266
2267 /* write the priority marks */
2268 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2269 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2270
7178d2a6
AD		 2271 /* save values for DPM */
2272 radeon_crtc->line_time = line_time;
2273 radeon_crtc->wm_high = latency_watermark_a;
2274 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2275}
2276
377edc8b
AD		 2277/**
2278 * evergreen_bandwidth_update - update display watermarks callback.
2279 *
2280 * @rdev: radeon_device pointer
2281 *
2282 * Update the display watermarks based on the requested mode(s)
2283 * (evergreen+).
2284 */
0ca2ab52 2285void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2286{
f9d9c362
AD		 2287 struct drm_display_mode *mode0 = NULL;
2288 struct drm_display_mode *mode1 = NULL;
2289 u32 num_heads = 0, lb_size;
2290 int i;
2291
2292 radeon_update_display_priority(rdev);
2293
2294 for (i = 0; i < rdev->num_crtc; i++) {
2295 if (rdev->mode_info.crtcs[i]->base.enabled)
2296 num_heads++;
2297 }
2298 for (i = 0; i < rdev->num_crtc; i += 2) {
2299 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2300 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2301 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2302 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2303 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2304 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2305 }
bcc1c2a1
AD		 2306}
2307
377edc8b
AD		 2308/**
2309 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2310 *
2311 * @rdev: radeon_device pointer
2312 *
2313 * Wait for the MC (memory controller) to be idle.
2314 * (evergreen+).
2315 * Returns 0 if the MC is idle, -1 if not.
2316 */
b9952a8a 2317int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2318{
2319 unsigned i;
2320 u32 tmp;
2321
2322 for (i = 0; i < rdev->usec_timeout; i++) {
2323 /* read MC_STATUS */
2324 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2325 if (!tmp)
2326 return 0;
2327 udelay(1);
2328 }
2329 return -1;
2330}
2331
2332/*
2333 * GART
2334 */
0fcdb61e
AD		 2335void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2336{
2337 unsigned i;
2338 u32 tmp;
2339
6f2f48a9
AD		 2340 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2341
0fcdb61e
AD		 2342 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2343 for (i = 0; i < rdev->usec_timeout; i++) {
2344 /* read MC_STATUS */
2345 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2346 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2347 if (tmp == 2) {
2348 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2349 return;
2350 }
2351 if (tmp) {
2352 return;
2353 }
2354 udelay(1);
2355 }
2356}
2357
1109ca09 2358static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2359{
2360 u32 tmp;
0fcdb61e 2361 int r;
bcc1c2a1 2362
c9a1be96 2363 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2364 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2365 return -EINVAL;
2366 }
2367 r = radeon_gart_table_vram_pin(rdev);
2368 if (r)
2369 return r;
82568565 2370 radeon_gart_restore(rdev);
bcc1c2a1
AD		 2371 /* Setup L2 cache */
2372 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2373 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2374 EFFECTIVE_L2_QUEUE_SIZE(7));
2375 WREG32(VM_L2_CNTL2, 0);
2376 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2377 /* Setup TLB control */
2378 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2379 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2380 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2381 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2382 if (rdev->flags & RADEON_IS_IGP) {
2383 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2384 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2385 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2386 } else {
2387 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2388 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2389 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2390 if ((rdev->family == CHIP_JUNIPER) ||
2391 (rdev->family == CHIP_CYPRESS) ||
2392 (rdev->family == CHIP_HEMLOCK) ||
2393 (rdev->family == CHIP_BARTS))
2394 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2395 }
bcc1c2a1
AD		 2396 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2397 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2398 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2399 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2400 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2401 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2402 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2403 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2404 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2405 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2406 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2407 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2408
0fcdb61e 2409 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2410 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2411 (unsigned)(rdev->mc.gtt_size >> 20),
2412 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2413 rdev->gart.ready = true;
2414 return 0;
2415}
2416
1109ca09 2417static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2418{
2419 u32 tmp;
bcc1c2a1
AD		 2420
2421 /* Disable all tables */
0fcdb61e
AD		 2422 WREG32(VM_CONTEXT0_CNTL, 0);
2423 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2424
2425 /* Setup L2 cache */
2426 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2427 EFFECTIVE_L2_QUEUE_SIZE(7));
2428 WREG32(VM_L2_CNTL2, 0);
2429 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2430 /* Setup TLB control */
2431 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2432 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2433 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2434 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2435 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2436 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2437 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2438 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2439 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2440}
2441
1109ca09 2442static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2443{
2444 evergreen_pcie_gart_disable(rdev);
2445 radeon_gart_table_vram_free(rdev);
2446 radeon_gart_fini(rdev);
2447}
2448
2449
1109ca09 2450static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2451{
2452 u32 tmp;
bcc1c2a1
AD		 2453
2454 /* Setup L2 cache */
2455 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2456 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2457 EFFECTIVE_L2_QUEUE_SIZE(7));
2458 WREG32(VM_L2_CNTL2, 0);
2459 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2460 /* Setup TLB control */
2461 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2462 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2463 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2464 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2465 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2466 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2467 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2468 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2469 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2470 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2471 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2472 WREG32(VM_CONTEXT0_CNTL, 0);
2473 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2474}
2475
b9952a8a 2476void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2477{
62444b74
AD		 2478 u32 crtc_enabled, tmp, frame_count, blackout;
2479 int i, j;
2480
5153550a
AD		 2481 if (!ASIC_IS_NODCE(rdev)) {
2482 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2483 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2484
5153550a
AD		 2485 /* disable VGA render */
2486 WREG32(VGA_RENDER_CONTROL, 0);
2487 }
62444b74
AD		 2488 /* blank the display controllers */
2489 for (i = 0; i < rdev->num_crtc; i++) {
2490 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2491 if (crtc_enabled) {
2492 save->crtc_enabled[i] = true;
2493 if (ASIC_IS_DCE6(rdev)) {
2494 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2495 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2496 radeon_wait_for_vblank(rdev, i);
abf1457b 2497 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2498 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2499 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2500 }
2501 } else {
2502 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2503 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2504 radeon_wait_for_vblank(rdev, i);
abf1457b 2505 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2506 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2507 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2508 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2509 }
2510 }
2511 /* wait for the next frame */
2512 frame_count = radeon_get_vblank_counter(rdev, i);
2513 for (j = 0; j < rdev->usec_timeout; j++) {
2514 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2515 break;
2516 udelay(1);
2517 }
abf1457b
AD		 2518
2519 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2520 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2521 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2522 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2523 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2524 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2525 save->crtc_enabled[i] = false;
2526 /* ***** */
804cc4a0
AD		 2527 } else {
2528 save->crtc_enabled[i] = false;
62444b74 2529 }
18007401 2530 }
bcc1c2a1 2531
62444b74
AD		 2532 radeon_mc_wait_for_idle(rdev);
2533
2534 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2535 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2536 /* Block CPU access */
2537 WREG32(BIF_FB_EN, 0);
2538 /* blackout the MC */
2539 blackout &= ~BLACKOUT_MODE_MASK;
2540 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2541 }
ed39fadd
AD		 2542 /* wait for the MC to settle */
2543 udelay(100);
968c0166
AD		 2544
2545 /* lock double buffered regs */
2546 for (i = 0; i < rdev->num_crtc; i++) {
2547 if (save->crtc_enabled[i]) {
2548 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2549 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2550 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2551 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2552 }
2553 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2554 if (!(tmp & 1)) {
2555 tmp |= 1;
2556 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2557 }
2558 }
2559 }
bcc1c2a1
AD		 2560}
2561
b9952a8a 2562void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2563{
62444b74
AD		 2564 u32 tmp, frame_count;
2565 int i, j;
18007401 2566
62444b74
AD		 2567 /* update crtc base addresses */
2568 for (i = 0; i < rdev->num_crtc; i++) {
2569 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2570 upper_32_bits(rdev->mc.vram_start));
62444b74 2571 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2572 upper_32_bits(rdev->mc.vram_start));
62444b74 2573 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2574 (u32)rdev->mc.vram_start);
62444b74 2575 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2576 (u32)rdev->mc.vram_start);
2577 }
5153550a
AD		 2578
2579 if (!ASIC_IS_NODCE(rdev)) {
2580 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2581 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2582 }
62444b74 2583
968c0166
AD		 2584 /* unlock regs and wait for update */
2585 for (i = 0; i < rdev->num_crtc; i++) {
2586 if (save->crtc_enabled[i]) {
2587 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2588 if ((tmp & 0x3) != 0) {
2589 tmp &= ~0x3;
2590 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2591 }
2592 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2593 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2594 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2595 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2596 }
2597 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2598 if (tmp & 1) {
2599 tmp &= ~1;
2600 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2601 }
2602 for (j = 0; j < rdev->usec_timeout; j++) {
2603 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2604 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2605 break;
2606 udelay(1);
2607 }
2608 }
2609 }
2610
62444b74
AD		 2611 /* unblackout the MC */
2612 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2613 tmp &= ~BLACKOUT_MODE_MASK;
2614 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2615 /* allow CPU access */
2616 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2617
2618 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2619 if (save->crtc_enabled[i]) {
62444b74
AD		 2620 if (ASIC_IS_DCE6(rdev)) {
2621 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2622 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2623 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2624 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2625 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2626 } else {
2627 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2628 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2629 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2630 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2631 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2632 }
2633 /* wait for the next frame */
2634 frame_count = radeon_get_vblank_counter(rdev, i);
2635 for (j = 0; j < rdev->usec_timeout; j++) {
2636 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2637 break;
2638 udelay(1);
2639 }
2640 }
2641 }
5153550a
AD		 2642 if (!ASIC_IS_NODCE(rdev)) {
2643 /* Unlock vga access */
2644 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2645 mdelay(1);
2646 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2647 }
bcc1c2a1
AD		 2648}
2649
755d819e 2650void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2651{
2652 struct evergreen_mc_save save;
2653 u32 tmp;
2654 int i, j;
2655
2656 /* Initialize HDP */
2657 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2658 WREG32((0x2c14 + j), 0x00000000);
2659 WREG32((0x2c18 + j), 0x00000000);
2660 WREG32((0x2c1c + j), 0x00000000);
2661 WREG32((0x2c20 + j), 0x00000000);
2662 WREG32((0x2c24 + j), 0x00000000);
2663 }
2664 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2665
2666 evergreen_mc_stop(rdev, &save);
2667 if (evergreen_mc_wait_for_idle(rdev)) {
2668 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2669 }
2670 /* Lockout access through VGA aperture*/
2671 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2672 /* Update configuration */
2673 if (rdev->flags & RADEON_IS_AGP) {
2674 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2675 /* VRAM before AGP */
2676 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2677 rdev->mc.vram_start >> 12);
2678 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2679 rdev->mc.gtt_end >> 12);
2680 } else {
2681 /* VRAM after AGP */
2682 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2683 rdev->mc.gtt_start >> 12);
2684 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2685 rdev->mc.vram_end >> 12);
2686 }
2687 } else {
2688 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2689 rdev->mc.vram_start >> 12);
2690 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2691 rdev->mc.vram_end >> 12);
2692 }
3b9832f6 2693 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2694 /* llano/ontario only */
2695 if ((rdev->family == CHIP_PALM) ||
2696 (rdev->family == CHIP_SUMO) ||
2697 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2698 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2699 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2700 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2701 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2702 }
bcc1c2a1
AD		 2703 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2704 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2705 WREG32(MC_VM_FB_LOCATION, tmp);
2706 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2707 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2708 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2709 if (rdev->flags & RADEON_IS_AGP) {
2710 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2711 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2712 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2713 } else {
2714 WREG32(MC_VM_AGP_BASE, 0);
2715 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2716 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2717 }
2718 if (evergreen_mc_wait_for_idle(rdev)) {
2719 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2720 }
2721 evergreen_mc_resume(rdev, &save);
2722 /* we need to own VRAM, so turn off the VGA renderer here
2723 * to stop it overwriting our objects */
2724 rv515_vga_render_disable(rdev);
2725}
2726
bcc1c2a1
AD		 2727/*
2728 * CP.
2729 */
12920591
AD		 2730void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2731{
876dc9f3 2732 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2733 u32 next_rptr;
7b1f2485 2734
12920591 2735 /* set to DX10/11 mode */
e32eb50d
CK		 2736 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2737 radeon_ring_write(ring, 1);
45df6803
CK		 2738
2739 if (ring->rptr_save_reg) {
89d35807 2740 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2741 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2742 radeon_ring_write(ring, ((ring->rptr_save_reg -
2743 PACKET3_SET_CONFIG_REG_START) >> 2));
2744 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2745 } else if (rdev->wb.enabled) {
2746 next_rptr = ring->wptr + 5 + 4;
2747 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2748 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2749 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2750 radeon_ring_write(ring, next_rptr);
2751 radeon_ring_write(ring, 0);
45df6803
CK		 2752 }
2753
e32eb50d
CK		 2754 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2755 radeon_ring_write(ring,
0f234f5f
AD		 2756#ifdef __BIG_ENDIAN
2757 (2 << 0) |
2758#endif
2759 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2760 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2761 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2762}
2763
bcc1c2a1
AD		 2764
2765static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2766{
fe251e2f
AD		 2767 const __be32 *fw_data;
2768 int i;
2769
2770 if (!rdev->me_fw || !rdev->pfp_fw)
2771 return -EINVAL;
bcc1c2a1 2772
fe251e2f 2773 r700_cp_stop(rdev);
0f234f5f
AD		 2774 WREG32(CP_RB_CNTL,
2775#ifdef __BIG_ENDIAN
2776 BUF_SWAP_32BIT |
2777#endif
2778 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2779
2780 fw_data = (const __be32 *)rdev->pfp_fw->data;
2781 WREG32(CP_PFP_UCODE_ADDR, 0);
2782 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2783 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2784 WREG32(CP_PFP_UCODE_ADDR, 0);
2785
2786 fw_data = (const __be32 *)rdev->me_fw->data;
2787 WREG32(CP_ME_RAM_WADDR, 0);
2788 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2789 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2790
2791 WREG32(CP_PFP_UCODE_ADDR, 0);
2792 WREG32(CP_ME_RAM_WADDR, 0);
2793 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2794 return 0;
2795}
2796
7e7b41d2
AD		 2797static int evergreen_cp_start(struct radeon_device *rdev)
2798{
e32eb50d 2799 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2800 int r, i;
7e7b41d2
AD		 2801 uint32_t cp_me;
2802
e32eb50d 2803 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2804 if (r) {
2805 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2806 return r;
2807 }
e32eb50d
CK		 2808 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2809 radeon_ring_write(ring, 0x1);
2810 radeon_ring_write(ring, 0x0);
2811 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2812 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2813 radeon_ring_write(ring, 0);
2814 radeon_ring_write(ring, 0);
2815 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2816
2817 cp_me = 0xff;
2818 WREG32(CP_ME_CNTL, cp_me);
2819
e32eb50d 2820 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2821 if (r) {
2822 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2823 return r;
2824 }
2281a378
AD		 2825
2826 /* setup clear context state */
e32eb50d
CK		 2827 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2828 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2829
2830 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2831 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2832
e32eb50d
CK		 2833 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2834 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2835
2836 /* set clear context state */
e32eb50d
CK		 2837 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2838 radeon_ring_write(ring, 0);
2281a378
AD		 2839
2840 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2841 radeon_ring_write(ring, 0xc0026f00);
2842 radeon_ring_write(ring, 0x00000000);
2843 radeon_ring_write(ring, 0x00000000);
2844 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2845
2846 /* Clear consts */
e32eb50d
CK		 2847 radeon_ring_write(ring, 0xc0036f00);
2848 radeon_ring_write(ring, 0x00000bc4);
2849 radeon_ring_write(ring, 0xffffffff);
2850 radeon_ring_write(ring, 0xffffffff);
2851 radeon_ring_write(ring, 0xffffffff);
2281a378 2852
e32eb50d
CK		 2853 radeon_ring_write(ring, 0xc0026900);
2854 radeon_ring_write(ring, 0x00000316);
2855 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2856 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2857
e32eb50d 2858 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2859
2860 return 0;
2861}
2862
1109ca09 2863static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2864{
e32eb50d 2865 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2866 u32 tmp;
2867 u32 rb_bufsz;
2868 int r;
2869
2870 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2871 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2872 SOFT_RESET_PA |
2873 SOFT_RESET_SH |
2874 SOFT_RESET_VGT |
a49a50da 2875 SOFT_RESET_SPI |
fe251e2f
AD		 2876 SOFT_RESET_SX));
2877 RREG32(GRBM_SOFT_RESET);
2878 mdelay(15);
2879 WREG32(GRBM_SOFT_RESET, 0);
2880 RREG32(GRBM_SOFT_RESET);
2881
2882 /* Set ring buffer size */
e32eb50d 2883 rb_bufsz = drm_order(ring->ring_size / 8);
724c80e1 2884 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2885#ifdef __BIG_ENDIAN
2886 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2887#endif
fe251e2f 2888 WREG32(CP_RB_CNTL, tmp);
15d3332f 2889 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2890 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2891
2892 /* Set the write pointer delay */
2893 WREG32(CP_RB_WPTR_DELAY, 0);
2894
2895 /* Initialize the ring buffer's read and write pointers */
2896 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2897 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2898 ring->wptr = 0;
2899 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2900
48fc7f7e 2901 /* set the wb address whether it's enabled or not */
0f234f5f 2902 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2903 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2904 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2905 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2906
2907 if (rdev->wb.enabled)
2908 WREG32(SCRATCH_UMSK, 0xff);
2909 else {
2910 tmp |= RB_NO_UPDATE;
2911 WREG32(SCRATCH_UMSK, 0);
2912 }
2913
fe251e2f
AD		 2914 mdelay(1);
2915 WREG32(CP_RB_CNTL, tmp);
2916
e32eb50d 2917 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2918 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2919
e32eb50d 2920 ring->rptr = RREG32(CP_RB_RPTR);
fe251e2f 2921
7e7b41d2 2922 evergreen_cp_start(rdev);
e32eb50d 2923 ring->ready = true;
f712812e 2924 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 2925 if (r) {
e32eb50d 2926 ring->ready = false;
fe251e2f
AD		 2927 return r;
2928 }
2929 return 0;
2930}
bcc1c2a1
AD		 2931
2932/*
2933 * Core functions
2934 */
bcc1c2a1
AD		 2935static void evergreen_gpu_init(struct radeon_device *rdev)
2936{
416a2bd2 2937 u32 gb_addr_config;
32fcdbf4 2938 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 2939 u32 sx_debug_1;
2940 u32 smx_dc_ctl0;
2941 u32 sq_config;
2942 u32 sq_lds_resource_mgmt;
2943 u32 sq_gpr_resource_mgmt_1;
2944 u32 sq_gpr_resource_mgmt_2;
2945 u32 sq_gpr_resource_mgmt_3;
2946 u32 sq_thread_resource_mgmt;
2947 u32 sq_thread_resource_mgmt_2;
2948 u32 sq_stack_resource_mgmt_1;
2949 u32 sq_stack_resource_mgmt_2;
2950 u32 sq_stack_resource_mgmt_3;
2951 u32 vgt_cache_invalidation;
f25a5c63 2952 u32 hdp_host_path_cntl, tmp;
416a2bd2 2953 u32 disabled_rb_mask;
32fcdbf4
AD		 2954 int i, j, num_shader_engines, ps_thread_count;
2955
2956 switch (rdev->family) {
2957 case CHIP_CYPRESS:
2958 case CHIP_HEMLOCK:
2959 rdev->config.evergreen.num_ses = 2;
2960 rdev->config.evergreen.max_pipes = 4;
2961 rdev->config.evergreen.max_tile_pipes = 8;
2962 rdev->config.evergreen.max_simds = 10;
2963 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2964 rdev->config.evergreen.max_gprs = 256;
2965 rdev->config.evergreen.max_threads = 248;
2966 rdev->config.evergreen.max_gs_threads = 32;
2967 rdev->config.evergreen.max_stack_entries = 512;
2968 rdev->config.evergreen.sx_num_of_sets = 4;
2969 rdev->config.evergreen.sx_max_export_size = 256;
2970 rdev->config.evergreen.sx_max_export_pos_size = 64;
2971 rdev->config.evergreen.sx_max_export_smx_size = 192;
2972 rdev->config.evergreen.max_hw_contexts = 8;
2973 rdev->config.evergreen.sq_num_cf_insts = 2;
2974
2975 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2976 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2977 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2978 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2979 break;
2980 case CHIP_JUNIPER:
2981 rdev->config.evergreen.num_ses = 1;
2982 rdev->config.evergreen.max_pipes = 4;
2983 rdev->config.evergreen.max_tile_pipes = 4;
2984 rdev->config.evergreen.max_simds = 10;
2985 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2986 rdev->config.evergreen.max_gprs = 256;
2987 rdev->config.evergreen.max_threads = 248;
2988 rdev->config.evergreen.max_gs_threads = 32;
2989 rdev->config.evergreen.max_stack_entries = 512;
2990 rdev->config.evergreen.sx_num_of_sets = 4;
2991 rdev->config.evergreen.sx_max_export_size = 256;
2992 rdev->config.evergreen.sx_max_export_pos_size = 64;
2993 rdev->config.evergreen.sx_max_export_smx_size = 192;
2994 rdev->config.evergreen.max_hw_contexts = 8;
2995 rdev->config.evergreen.sq_num_cf_insts = 2;
2996
2997 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2998 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2999 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3000 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3001 break;
3002 case CHIP_REDWOOD:
3003 rdev->config.evergreen.num_ses = 1;
3004 rdev->config.evergreen.max_pipes = 4;
3005 rdev->config.evergreen.max_tile_pipes = 4;
3006 rdev->config.evergreen.max_simds = 5;
3007 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3008 rdev->config.evergreen.max_gprs = 256;
3009 rdev->config.evergreen.max_threads = 248;
3010 rdev->config.evergreen.max_gs_threads = 32;
3011 rdev->config.evergreen.max_stack_entries = 256;
3012 rdev->config.evergreen.sx_num_of_sets = 4;
3013 rdev->config.evergreen.sx_max_export_size = 256;
3014 rdev->config.evergreen.sx_max_export_pos_size = 64;
3015 rdev->config.evergreen.sx_max_export_smx_size = 192;
3016 rdev->config.evergreen.max_hw_contexts = 8;
3017 rdev->config.evergreen.sq_num_cf_insts = 2;
3018
3019 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3020 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3021 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3022 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3023 break;
3024 case CHIP_CEDAR:
3025 default:
3026 rdev->config.evergreen.num_ses = 1;
3027 rdev->config.evergreen.max_pipes = 2;
3028 rdev->config.evergreen.max_tile_pipes = 2;
3029 rdev->config.evergreen.max_simds = 2;
3030 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3031 rdev->config.evergreen.max_gprs = 256;
3032 rdev->config.evergreen.max_threads = 192;
3033 rdev->config.evergreen.max_gs_threads = 16;
3034 rdev->config.evergreen.max_stack_entries = 256;
3035 rdev->config.evergreen.sx_num_of_sets = 4;
3036 rdev->config.evergreen.sx_max_export_size = 128;
3037 rdev->config.evergreen.sx_max_export_pos_size = 32;
3038 rdev->config.evergreen.sx_max_export_smx_size = 96;
3039 rdev->config.evergreen.max_hw_contexts = 4;
3040 rdev->config.evergreen.sq_num_cf_insts = 1;
3041
d5e455e4
AD		 3042 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3043 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3044 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3045 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3046 break;
3047 case CHIP_PALM:
3048 rdev->config.evergreen.num_ses = 1;
3049 rdev->config.evergreen.max_pipes = 2;
3050 rdev->config.evergreen.max_tile_pipes = 2;
3051 rdev->config.evergreen.max_simds = 2;
3052 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3053 rdev->config.evergreen.max_gprs = 256;
3054 rdev->config.evergreen.max_threads = 192;
3055 rdev->config.evergreen.max_gs_threads = 16;
3056 rdev->config.evergreen.max_stack_entries = 256;
3057 rdev->config.evergreen.sx_num_of_sets = 4;
3058 rdev->config.evergreen.sx_max_export_size = 128;
3059 rdev->config.evergreen.sx_max_export_pos_size = 32;
3060 rdev->config.evergreen.sx_max_export_smx_size = 96;
3061 rdev->config.evergreen.max_hw_contexts = 4;
3062 rdev->config.evergreen.sq_num_cf_insts = 1;
3063
d5c5a72f
AD		 3064 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3065 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3066 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3067 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3068 break;
3069 case CHIP_SUMO:
3070 rdev->config.evergreen.num_ses = 1;
3071 rdev->config.evergreen.max_pipes = 4;
bd25f078 3072 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3073 if (rdev->pdev->device == 0x9648)
3074 rdev->config.evergreen.max_simds = 3;
3075 else if ((rdev->pdev->device == 0x9647) ||
3076 (rdev->pdev->device == 0x964a))
3077 rdev->config.evergreen.max_simds = 4;
3078 else
3079 rdev->config.evergreen.max_simds = 5;
3080 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3081 rdev->config.evergreen.max_gprs = 256;
3082 rdev->config.evergreen.max_threads = 248;
3083 rdev->config.evergreen.max_gs_threads = 32;
3084 rdev->config.evergreen.max_stack_entries = 256;
3085 rdev->config.evergreen.sx_num_of_sets = 4;
3086 rdev->config.evergreen.sx_max_export_size = 256;
3087 rdev->config.evergreen.sx_max_export_pos_size = 64;
3088 rdev->config.evergreen.sx_max_export_smx_size = 192;
3089 rdev->config.evergreen.max_hw_contexts = 8;
3090 rdev->config.evergreen.sq_num_cf_insts = 2;
3091
3092 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3093 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3094 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3095 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3096 break;
3097 case CHIP_SUMO2:
3098 rdev->config.evergreen.num_ses = 1;
3099 rdev->config.evergreen.max_pipes = 4;
3100 rdev->config.evergreen.max_tile_pipes = 4;
3101 rdev->config.evergreen.max_simds = 2;
3102 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3103 rdev->config.evergreen.max_gprs = 256;
3104 rdev->config.evergreen.max_threads = 248;
3105 rdev->config.evergreen.max_gs_threads = 32;
3106 rdev->config.evergreen.max_stack_entries = 512;
3107 rdev->config.evergreen.sx_num_of_sets = 4;
3108 rdev->config.evergreen.sx_max_export_size = 256;
3109 rdev->config.evergreen.sx_max_export_pos_size = 64;
3110 rdev->config.evergreen.sx_max_export_smx_size = 192;
3111 rdev->config.evergreen.max_hw_contexts = 8;
3112 rdev->config.evergreen.sq_num_cf_insts = 2;
3113
adb68fa2
AD		 3114 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3115 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3116 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3117 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3118 break;
3119 case CHIP_BARTS:
3120 rdev->config.evergreen.num_ses = 2;
3121 rdev->config.evergreen.max_pipes = 4;
3122 rdev->config.evergreen.max_tile_pipes = 8;
3123 rdev->config.evergreen.max_simds = 7;
3124 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3125 rdev->config.evergreen.max_gprs = 256;
3126 rdev->config.evergreen.max_threads = 248;
3127 rdev->config.evergreen.max_gs_threads = 32;
3128 rdev->config.evergreen.max_stack_entries = 512;
3129 rdev->config.evergreen.sx_num_of_sets = 4;
3130 rdev->config.evergreen.sx_max_export_size = 256;
3131 rdev->config.evergreen.sx_max_export_pos_size = 64;
3132 rdev->config.evergreen.sx_max_export_smx_size = 192;
3133 rdev->config.evergreen.max_hw_contexts = 8;
3134 rdev->config.evergreen.sq_num_cf_insts = 2;
3135
3136 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3137 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3138 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3139 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3140 break;
3141 case CHIP_TURKS:
3142 rdev->config.evergreen.num_ses = 1;
3143 rdev->config.evergreen.max_pipes = 4;
3144 rdev->config.evergreen.max_tile_pipes = 4;
3145 rdev->config.evergreen.max_simds = 6;
3146 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3147 rdev->config.evergreen.max_gprs = 256;
3148 rdev->config.evergreen.max_threads = 248;
3149 rdev->config.evergreen.max_gs_threads = 32;
3150 rdev->config.evergreen.max_stack_entries = 256;
3151 rdev->config.evergreen.sx_num_of_sets = 4;
3152 rdev->config.evergreen.sx_max_export_size = 256;
3153 rdev->config.evergreen.sx_max_export_pos_size = 64;
3154 rdev->config.evergreen.sx_max_export_smx_size = 192;
3155 rdev->config.evergreen.max_hw_contexts = 8;
3156 rdev->config.evergreen.sq_num_cf_insts = 2;
3157
3158 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3159 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3160 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3161 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3162 break;
3163 case CHIP_CAICOS:
3164 rdev->config.evergreen.num_ses = 1;
bd25f078 3165 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3166 rdev->config.evergreen.max_tile_pipes = 2;
3167 rdev->config.evergreen.max_simds = 2;
3168 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3169 rdev->config.evergreen.max_gprs = 256;
3170 rdev->config.evergreen.max_threads = 192;
3171 rdev->config.evergreen.max_gs_threads = 16;
3172 rdev->config.evergreen.max_stack_entries = 256;
3173 rdev->config.evergreen.sx_num_of_sets = 4;
3174 rdev->config.evergreen.sx_max_export_size = 128;
3175 rdev->config.evergreen.sx_max_export_pos_size = 32;
3176 rdev->config.evergreen.sx_max_export_smx_size = 96;
3177 rdev->config.evergreen.max_hw_contexts = 4;
3178 rdev->config.evergreen.sq_num_cf_insts = 1;
3179
32fcdbf4
AD		 3180 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3181 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3182 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3183 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3184 break;
3185 }
3186
3187 /* Initialize HDP */
3188 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3189 WREG32((0x2c14 + j), 0x00000000);
3190 WREG32((0x2c18 + j), 0x00000000);
3191 WREG32((0x2c1c + j), 0x00000000);
3192 WREG32((0x2c20 + j), 0x00000000);
3193 WREG32((0x2c24 + j), 0x00000000);
3194 }
3195
3196 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3197
d054ac16
AD		 3198 evergreen_fix_pci_max_read_req_size(rdev);
3199
32fcdbf4 3200 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3201 if ((rdev->family == CHIP_PALM) ||
3202 (rdev->family == CHIP_SUMO) ||
3203 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3204 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3205 else
3206 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3207
1aa52bd3
AD		 3208 /* setup tiling info dword. gb_addr_config is not adequate since it does
3209 * not have bank info, so create a custom tiling dword.
3210 * bits 3:0 num_pipes
3211 * bits 7:4 num_banks
3212 * bits 11:8 group_size
3213 * bits 15:12 row_size
3214 */
3215 rdev->config.evergreen.tile_config = 0;
3216 switch (rdev->config.evergreen.max_tile_pipes) {
3217 case 1:
3218 default:
3219 rdev->config.evergreen.tile_config |= (0 << 0);
3220 break;
3221 case 2:
3222 rdev->config.evergreen.tile_config |= (1 << 0);
3223 break;
3224 case 4:
3225 rdev->config.evergreen.tile_config |= (2 << 0);
3226 break;
3227 case 8:
3228 rdev->config.evergreen.tile_config |= (3 << 0);
3229 break;
3230 }
d698a34d 3231 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3232 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3233 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3234 else {
c8d15edc
AD		 3235 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3236 case 0: /* four banks */
29d65406 3237 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3238 break;
3239 case 1: /* eight banks */
3240 rdev->config.evergreen.tile_config |= 1 << 4;
3241 break;
3242 case 2: /* sixteen banks */
3243 default:
3244 rdev->config.evergreen.tile_config |= 2 << 4;
3245 break;
3246 }
29d65406 3247 }
416a2bd2 3248 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3249 rdev->config.evergreen.tile_config |=
3250 ((gb_addr_config & 0x30000000) >> 28) << 12;
3251
416a2bd2 3252 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
32fcdbf4 3253
416a2bd2
AD		 3254 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3255 u32 efuse_straps_4;
3256 u32 efuse_straps_3;
32fcdbf4 3257
ff82bbc4
AD		 3258 efuse_straps_4 = RREG32_RCU(0x204);
3259 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3260 tmp = (((efuse_straps_4 & 0xf) << 4) |
3261 ((efuse_straps_3 & 0xf0000000) >> 28));
3262 } else {
3263 tmp = 0;
3264 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3265 u32 rb_disable_bitmap;
3266
3267 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3268 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3269 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3270 tmp <<= 4;
3271 tmp |= rb_disable_bitmap;
32fcdbf4 3272 }
416a2bd2
AD		 3273 }
3274 /* enabled rb are just the one not disabled :) */
3275 disabled_rb_mask = tmp;
cedb655a
AD		 3276 tmp = 0;
3277 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3278 tmp |= (1 << i);
3279 /* if all the backends are disabled, fix it up here */
3280 if ((disabled_rb_mask & tmp) == tmp) {
3281 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3282 disabled_rb_mask &= ~(1 << i);
3283 }
32fcdbf4 3284
416a2bd2
AD		 3285 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3286 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3287
416a2bd2
AD		 3288 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3289 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3290 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3291 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3292 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3293 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3294 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3295
f7eb9730
AD		 3296 if ((rdev->config.evergreen.max_backends == 1) &&
3297 (rdev->flags & RADEON_IS_IGP)) {
3298 if ((disabled_rb_mask & 3) == 1) {
3299 /* RB0 disabled, RB1 enabled */
3300 tmp = 0x11111111;
3301 } else {
3302 /* RB1 disabled, RB0 enabled */
3303 tmp = 0x00000000;
3304 }
3305 } else {
3306 tmp = gb_addr_config & NUM_PIPES_MASK;
3307 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3308 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3309 }
416a2bd2 3310 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3311
3312 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3313 WREG32(CGTS_TCC_DISABLE, 0);
3314 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3315 WREG32(CGTS_USER_TCC_DISABLE, 0);
3316
3317 /* set HW defaults for 3D engine */
3318 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3319 ROQ_IB2_START(0x2b)));
3320
3321 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3322
3323 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3324 SYNC_GRADIENT |
3325 SYNC_WALKER |
3326 SYNC_ALIGNER));
3327
3328 sx_debug_1 = RREG32(SX_DEBUG_1);
3329 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3330 WREG32(SX_DEBUG_1, sx_debug_1);
3331
3332
3333 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3334 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3335 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3336 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3337
b866d133
AD		 3338 if (rdev->family <= CHIP_SUMO2)
3339 WREG32(SMX_SAR_CTL0, 0x00010000);
3340
32fcdbf4
AD		 3341 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3342 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3343 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3344
3345 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3346 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3347 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3348
3349 WREG32(VGT_NUM_INSTANCES, 1);
3350 WREG32(SPI_CONFIG_CNTL, 0);
3351 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3352 WREG32(CP_PERFMON_CNTL, 0);
3353
3354 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3355 FETCH_FIFO_HIWATER(0x4) |
3356 DONE_FIFO_HIWATER(0xe0) |
3357 ALU_UPDATE_FIFO_HIWATER(0x8)));
3358
3359 sq_config = RREG32(SQ_CONFIG);
3360 sq_config &= ~(PS_PRIO(3) |
3361 VS_PRIO(3) |
3362 GS_PRIO(3) |
3363 ES_PRIO(3));
3364 sq_config |= (VC_ENABLE |
3365 EXPORT_SRC_C |
3366 PS_PRIO(0) |
3367 VS_PRIO(1) |
3368 GS_PRIO(2) |
3369 ES_PRIO(3));
3370
d5e455e4
AD		 3371 switch (rdev->family) {
3372 case CHIP_CEDAR:
3373 case CHIP_PALM:
d5c5a72f
AD		 3374 case CHIP_SUMO:
3375 case CHIP_SUMO2:
adb68fa2 3376 case CHIP_CAICOS:
32fcdbf4
AD		 3377 /* no vertex cache */
3378 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3379 break;
3380 default:
3381 break;
3382 }
32fcdbf4
AD		 3383
3384 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3385
3386 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3387 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3388 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3389 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3390 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3391 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3392 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3393
d5e455e4
AD		 3394 switch (rdev->family) {
3395 case CHIP_CEDAR:
3396 case CHIP_PALM:
d5c5a72f
AD		 3397 case CHIP_SUMO:
3398 case CHIP_SUMO2:
32fcdbf4 3399 ps_thread_count = 96;
d5e455e4
AD		 3400 break;
3401 default:
32fcdbf4 3402 ps_thread_count = 128;
d5e455e4
AD		 3403 break;
3404 }
32fcdbf4
AD		 3405
3406 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3407 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3408 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3409 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3410 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3411 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3412
3413 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3414 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3415 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3416 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3417 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3418 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3419
3420 WREG32(SQ_CONFIG, sq_config);
3421 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3422 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3423 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3424 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3425 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3426 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3427 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3428 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3429 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3430 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3431
3432 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3433 FORCE_EOV_MAX_REZ_CNT(255)));
3434
d5e455e4
AD		 3435 switch (rdev->family) {
3436 case CHIP_CEDAR:
3437 case CHIP_PALM:
d5c5a72f
AD		 3438 case CHIP_SUMO:
3439 case CHIP_SUMO2:
adb68fa2 3440 case CHIP_CAICOS:
32fcdbf4 3441 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3442 break;
3443 default:
32fcdbf4 3444 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3445 break;
3446 }
32fcdbf4
AD		 3447 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3448 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3449
3450 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3451 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3452 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3453
60a4a3e0
AD		 3454 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3455 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3456
32fcdbf4
AD		 3457 WREG32(CB_PERF_CTR0_SEL_0, 0);
3458 WREG32(CB_PERF_CTR0_SEL_1, 0);
3459 WREG32(CB_PERF_CTR1_SEL_0, 0);
3460 WREG32(CB_PERF_CTR1_SEL_1, 0);
3461 WREG32(CB_PERF_CTR2_SEL_0, 0);
3462 WREG32(CB_PERF_CTR2_SEL_1, 0);
3463 WREG32(CB_PERF_CTR3_SEL_0, 0);
3464 WREG32(CB_PERF_CTR3_SEL_1, 0);
3465
60a4a3e0
AD		 3466 /* clear render buffer base addresses */
3467 WREG32(CB_COLOR0_BASE, 0);
3468 WREG32(CB_COLOR1_BASE, 0);
3469 WREG32(CB_COLOR2_BASE, 0);
3470 WREG32(CB_COLOR3_BASE, 0);
3471 WREG32(CB_COLOR4_BASE, 0);
3472 WREG32(CB_COLOR5_BASE, 0);
3473 WREG32(CB_COLOR6_BASE, 0);
3474 WREG32(CB_COLOR7_BASE, 0);
3475 WREG32(CB_COLOR8_BASE, 0);
3476 WREG32(CB_COLOR9_BASE, 0);
3477 WREG32(CB_COLOR10_BASE, 0);
3478 WREG32(CB_COLOR11_BASE, 0);
3479
3480 /* set the shader const cache sizes to 0 */
3481 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3482 WREG32(i, 0);
3483 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3484 WREG32(i, 0);
3485
f25a5c63
AD		 3486 tmp = RREG32(HDP_MISC_CNTL);
3487 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3488 WREG32(HDP_MISC_CNTL, tmp);
3489
32fcdbf4
AD		 3490 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3491 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3492
3493 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3494
3495 udelay(50);
3496
bcc1c2a1
AD		 3497}
3498
3499int evergreen_mc_init(struct radeon_device *rdev)
3500{
bcc1c2a1
AD		 3501 u32 tmp;
3502 int chansize, numchan;
bcc1c2a1
AD		 3503
3504 /* Get VRAM informations */
3505 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3506 if ((rdev->family == CHIP_PALM) ||
3507 (rdev->family == CHIP_SUMO) ||
3508 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3509 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3510 else
3511 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3512 if (tmp & CHANSIZE_OVERRIDE) {
3513 chansize = 16;
3514 } else if (tmp & CHANSIZE_MASK) {
3515 chansize = 64;
3516 } else {
3517 chansize = 32;
3518 }
3519 tmp = RREG32(MC_SHARED_CHMAP);
3520 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3521 case 0:
3522 default:
3523 numchan = 1;
3524 break;
3525 case 1:
3526 numchan = 2;
3527 break;
3528 case 2:
3529 numchan = 4;
3530 break;
3531 case 3:
3532 numchan = 8;
3533 break;
3534 }
3535 rdev->mc.vram_width = numchan * chansize;
3536 /* Could aper size report 0 ? */
01d73a69
JC		 3537 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3538 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3539 /* Setup GPU memory space */
05b3ef69
AD		 3540 if ((rdev->family == CHIP_PALM) ||
3541 (rdev->family == CHIP_SUMO) ||
3542 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3543 /* size in bytes on fusion */
3544 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3545 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3546 } else {
05b3ef69 3547 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3548 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3549 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3550 }
51e5fcd3 3551 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3552 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3553 radeon_update_bandwidth_info(rdev);
3554
bcc1c2a1
AD		 3555 return 0;
3556}
d594e46a 3557
187e3593 3558void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3559{
64c56e8c 3560 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3561 RREG32(GRBM_STATUS));
64c56e8c 3562 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3563 RREG32(GRBM_STATUS_SE0));
64c56e8c 3564 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3565 RREG32(GRBM_STATUS_SE1));
64c56e8c 3566 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3567 RREG32(SRBM_STATUS));
a65a4369
AD		 3568 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3569 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3570 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3571 RREG32(CP_STALLED_STAT1));
3572 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3573 RREG32(CP_STALLED_STAT2));
3574 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3575 RREG32(CP_BUSY_STAT));
3576 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3577 RREG32(CP_STAT));
eaaa6983
JG		 3578 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3579 RREG32(DMA_STATUS_REG));
168757ea
AD		 3580 if (rdev->family >= CHIP_CAYMAN) {
3581 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3582 RREG32(DMA_STATUS_REG + 0x800));
3583 }
0ecebb9e
AD		 3584}
3585
168757ea 3586bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3587{
a65a4369
AD		 3588 u32 crtc_hung = 0;
3589 u32 crtc_status[6];
3590 u32 i, j, tmp;
3591
3592 for (i = 0; i < rdev->num_crtc; i++) {
3593 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3594 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3595 crtc_hung |= (1 << i);
3596 }
3597 }
3598
3599 for (j = 0; j < 10; j++) {
3600 for (i = 0; i < rdev->num_crtc; i++) {
3601 if (crtc_hung & (1 << i)) {
3602 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3603 if (tmp != crtc_status[i])
3604 crtc_hung &= ~(1 << i);
3605 }
3606 }
3607 if (crtc_hung == 0)
3608 return false;
3609 udelay(100);
3610 }
3611
3612 return true;
3613}
3614
3615static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3616{
3617 u32 reset_mask = 0;
b7630473 3618 u32 tmp;
0ecebb9e 3619
a65a4369
AD		 3620 /* GRBM_STATUS */
3621 tmp = RREG32(GRBM_STATUS);
3622 if (tmp & (PA_BUSY | SC_BUSY |
3623 SH_BUSY | SX_BUSY |
3624 TA_BUSY | VGT_BUSY |
3625 DB_BUSY | CB_BUSY |
3626 SPI_BUSY | VGT_BUSY_NO_DMA))
3627 reset_mask |= RADEON_RESET_GFX;
3628
3629 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3630 CP_BUSY | CP_COHERENCY_BUSY))
3631 reset_mask |= RADEON_RESET_CP;
3632
3633 if (tmp & GRBM_EE_BUSY)
3634 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3635
a65a4369
AD		 3636 /* DMA_STATUS_REG */
3637 tmp = RREG32(DMA_STATUS_REG);
3638 if (!(tmp & DMA_IDLE))
3639 reset_mask |= RADEON_RESET_DMA;
3640
3641 /* SRBM_STATUS2 */
3642 tmp = RREG32(SRBM_STATUS2);
3643 if (tmp & DMA_BUSY)
3644 reset_mask |= RADEON_RESET_DMA;
3645
3646 /* SRBM_STATUS */
3647 tmp = RREG32(SRBM_STATUS);
3648 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3649 reset_mask |= RADEON_RESET_RLC;
3650
3651 if (tmp & IH_BUSY)
3652 reset_mask |= RADEON_RESET_IH;
3653
3654 if (tmp & SEM_BUSY)
3655 reset_mask |= RADEON_RESET_SEM;
3656
3657 if (tmp & GRBM_RQ_PENDING)
3658 reset_mask |= RADEON_RESET_GRBM;
3659
3660 if (tmp & VMC_BUSY)
3661 reset_mask |= RADEON_RESET_VMC;
3662
3663 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3664 MCC_BUSY | MCD_BUSY))
3665 reset_mask |= RADEON_RESET_MC;
3666
3667 if (evergreen_is_display_hung(rdev))
3668 reset_mask |= RADEON_RESET_DISPLAY;
3669
3670 /* VM_L2_STATUS */
3671 tmp = RREG32(VM_L2_STATUS);
3672 if (tmp & L2_BUSY)
3673 reset_mask |= RADEON_RESET_VMC;
3674
d808fc88
AD		 3675 /* Skip MC reset as it's mostly likely not hung, just busy */
3676 if (reset_mask & RADEON_RESET_MC) {
3677 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3678 reset_mask &= ~RADEON_RESET_MC;
3679 }
3680
a65a4369
AD		 3681 return reset_mask;
3682}
3683
3684static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3685{
3686 struct evergreen_mc_save save;
3687 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3688 u32 tmp;
19fc42ed 3689
0ecebb9e 3690 if (reset_mask == 0)
a65a4369 3691 return;
0ecebb9e
AD		 3692
3693 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3694
b7630473
AD		 3695 evergreen_print_gpu_status_regs(rdev);
3696
b7630473
AD		 3697 /* Disable CP parsing/prefetching */
3698 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3699
3700 if (reset_mask & RADEON_RESET_DMA) {
3701 /* Disable DMA */
3702 tmp = RREG32(DMA_RB_CNTL);
3703 tmp &= ~DMA_RB_ENABLE;
3704 WREG32(DMA_RB_CNTL, tmp);
3705 }
3706
b21b6e7a
AD		 3707 udelay(50);
3708
3709 evergreen_mc_stop(rdev, &save);
3710 if (evergreen_mc_wait_for_idle(rdev)) {
3711 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3712 }
3713
b7630473
AD		 3714 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3715 grbm_soft_reset |= SOFT_RESET_DB |
3716 SOFT_RESET_CB |
3717 SOFT_RESET_PA |
3718 SOFT_RESET_SC |
3719 SOFT_RESET_SPI |
3720 SOFT_RESET_SX |
3721 SOFT_RESET_SH |
3722 SOFT_RESET_TC |
3723 SOFT_RESET_TA |
3724 SOFT_RESET_VC |
3725 SOFT_RESET_VGT;
3726 }
3727
3728 if (reset_mask & RADEON_RESET_CP) {
3729 grbm_soft_reset |= SOFT_RESET_CP |
3730 SOFT_RESET_VGT;
3731
3732 srbm_soft_reset |= SOFT_RESET_GRBM;
3733 }
0ecebb9e
AD		 3734
3735 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3736 srbm_soft_reset |= SOFT_RESET_DMA;
3737
a65a4369
AD		 3738 if (reset_mask & RADEON_RESET_DISPLAY)
3739 srbm_soft_reset |= SOFT_RESET_DC;
3740
3741 if (reset_mask & RADEON_RESET_RLC)
3742 srbm_soft_reset |= SOFT_RESET_RLC;
3743
3744 if (reset_mask & RADEON_RESET_SEM)
3745 srbm_soft_reset |= SOFT_RESET_SEM;
3746
3747 if (reset_mask & RADEON_RESET_IH)
3748 srbm_soft_reset |= SOFT_RESET_IH;
3749
3750 if (reset_mask & RADEON_RESET_GRBM)
3751 srbm_soft_reset |= SOFT_RESET_GRBM;
3752
3753 if (reset_mask & RADEON_RESET_VMC)
3754 srbm_soft_reset |= SOFT_RESET_VMC;
3755
24178ec4
AD		 3756 if (!(rdev->flags & RADEON_IS_IGP)) {
3757 if (reset_mask & RADEON_RESET_MC)
3758 srbm_soft_reset |= SOFT_RESET_MC;
3759 }
a65a4369 3760
b7630473
AD		 3761 if (grbm_soft_reset) {
3762 tmp = RREG32(GRBM_SOFT_RESET);
3763 tmp |= grbm_soft_reset;
3764 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3765 WREG32(GRBM_SOFT_RESET, tmp);
3766 tmp = RREG32(GRBM_SOFT_RESET);
3767
3768 udelay(50);
3769
3770 tmp &= ~grbm_soft_reset;
3771 WREG32(GRBM_SOFT_RESET, tmp);
3772 tmp = RREG32(GRBM_SOFT_RESET);
3773 }
3774
3775 if (srbm_soft_reset) {
3776 tmp = RREG32(SRBM_SOFT_RESET);
3777 tmp |= srbm_soft_reset;
3778 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3779 WREG32(SRBM_SOFT_RESET, tmp);
3780 tmp = RREG32(SRBM_SOFT_RESET);
3781
3782 udelay(50);
3783
3784 tmp &= ~srbm_soft_reset;
3785 WREG32(SRBM_SOFT_RESET, tmp);
3786 tmp = RREG32(SRBM_SOFT_RESET);
3787 }
0ecebb9e
AD		 3788
3789 /* Wait a little for things to settle down */
3790 udelay(50);
3791
747943ea 3792 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3793 udelay(50);
3794
b7630473 3795 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3796}
3797
a2d07b74 3798int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3799{
a65a4369
AD		 3800 u32 reset_mask;
3801
3802 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3803
3804 if (reset_mask)
3805 r600_set_bios_scratch_engine_hung(rdev, true);
3806
3807 evergreen_gpu_soft_reset(rdev, reset_mask);
3808
3809 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3810
3811 if (!reset_mask)
3812 r600_set_bios_scratch_engine_hung(rdev, false);
3813
3814 return 0;
747943ea
AD		 3815}
3816
123bc183
AD		 3817/**
3818 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3819 *
3820 * @rdev: radeon_device pointer
3821 * @ring: radeon_ring structure holding ring information
3822 *
3823 * Check if the GFX engine is locked up.
3824 * Returns true if the engine appears to be locked up, false if not.
3825 */
3826bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3827{
3828 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3829
3830 if (!(reset_mask & (RADEON_RESET_GFX |
3831 RADEON_RESET_COMPUTE |
3832 RADEON_RESET_CP))) {
3833 radeon_ring_lockup_update(ring);
3834 return false;
3835 }
3836 /* force CP activities */
3837 radeon_ring_force_activity(rdev, ring);
3838 return radeon_ring_test_lockup(rdev, ring);
3839}
3840
3841/**
3842 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3843 *
3844 * @rdev: radeon_device pointer
3845 * @ring: radeon_ring structure holding ring information
3846 *
3847 * Check if the async DMA engine is locked up.
3848 * Returns true if the engine appears to be locked up, false if not.
3849 */
3850bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3851{
3852 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3853
3854 if (!(reset_mask & RADEON_RESET_DMA)) {
3855 radeon_ring_lockup_update(ring);
3856 return false;
3857 }
3858 /* force ring activities */
3859 radeon_ring_force_activity(rdev, ring);
3860 return radeon_ring_test_lockup(rdev, ring);
3861}
3862
2948f5e6
AD		 3863/*
3864 * RLC
3865 */
3866#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3867#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3868
3869void sumo_rlc_fini(struct radeon_device *rdev)
3870{
3871 int r;
3872
3873 /* save restore block */
3874 if (rdev->rlc.save_restore_obj) {
3875 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3876 if (unlikely(r != 0))
3877 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3878 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3879 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3880
3881 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3882 rdev->rlc.save_restore_obj = NULL;
3883 }
3884
3885 /* clear state block */
3886 if (rdev->rlc.clear_state_obj) {
3887 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3888 if (unlikely(r != 0))
3889 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3890 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3891 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3892
3893 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3894 rdev->rlc.clear_state_obj = NULL;
3895 }
3896}
3897
3898int sumo_rlc_init(struct radeon_device *rdev)
3899{
1fd11777 3900 const u32 *src_ptr;
2948f5e6
AD		 3901 volatile u32 *dst_ptr;
3902 u32 dws, data, i, j, k, reg_num;
3903 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
3904 u64 reg_list_mc_addr;
1fd11777 3905 const struct cs_section_def *cs_data;
2948f5e6
AD		 3906 int r;
3907
3908 src_ptr = rdev->rlc.reg_list;
3909 dws = rdev->rlc.reg_list_size;
3910 cs_data = rdev->rlc.cs_data;
3911
10b7ca7e
AD		 3912 if (src_ptr) {
3913 /* save restore block */
3914 if (rdev->rlc.save_restore_obj == NULL) {
3915 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3916 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3917 if (r) {
3918 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3919 return r;
3920 }
3921 }
3922
3923 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3924 if (unlikely(r != 0)) {
3925 sumo_rlc_fini(rdev);
3926 return r;
3927 }
3928 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3929 &rdev->rlc.save_restore_gpu_addr);
2948f5e6 3930 if (r) {
10b7ca7e
AD		 3931 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3932 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3933 sumo_rlc_fini(rdev);
2948f5e6
AD		 3934 return r;
3935 }
2948f5e6 3936
10b7ca7e
AD		 3937 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3938 if (r) {
3939 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3940 sumo_rlc_fini(rdev);
3941 return r;
3942 }
3943 /* write the sr buffer */
3944 dst_ptr = rdev->rlc.sr_ptr;
1fd11777
AD		 3945 if (rdev->family >= CHIP_TAHITI) {
3946 /* SI */
3947 for (i = 0; i < dws; i++)
3948 dst_ptr[i] = src_ptr[i];
3949 } else {
3950 /* ON/LN/TN */
3951 /* format:
3952 * dw0: (reg2 << 16) | reg1
3953 * dw1: reg1 save space
3954 * dw2: reg2 save space
3955 */
3956 for (i = 0; i < dws; i++) {
3957 data = src_ptr[i] >> 2;
3958 i++;
3959 if (i < dws)
3960 data |= (src_ptr[i] >> 2) << 16;
3961 j = (((i - 1) * 3) / 2);
3962 dst_ptr[j] = data;
3963 }
3964 j = ((i * 3) / 2);
3965 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
10b7ca7e 3966 }
10b7ca7e 3967 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
2948f5e6 3968 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2948f5e6 3969 }
2948f5e6 3970
10b7ca7e
AD		 3971 if (cs_data) {
3972 /* clear state block */
3973 reg_list_num = 0;
3974 dws = 0;
3975 for (i = 0; cs_data[i].section != NULL; i++) {
3976 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3977 reg_list_num++;
3978 dws += cs_data[i].section[j].reg_count;
3979 }
2948f5e6 3980 }
10b7ca7e
AD		 3981 reg_list_blk_index = (3 * reg_list_num + 2);
3982 dws += reg_list_blk_index;
3983
3984 if (rdev->rlc.clear_state_obj == NULL) {
3985 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3986 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
3987 if (r) {
3988 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
3989 sumo_rlc_fini(rdev);
3990 return r;
3991 }
3992 }
3993 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3994 if (unlikely(r != 0)) {
3995 sumo_rlc_fini(rdev);
3996 return r;
3997 }
3998 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
3999 &rdev->rlc.clear_state_gpu_addr);
2948f5e6 4000 if (r) {
10b7ca7e
AD		 4001 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4002 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
2948f5e6
AD		 4003 sumo_rlc_fini(rdev);
4004 return r;
4005 }
2948f5e6 4006
10b7ca7e
AD		 4007 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4008 if (r) {
4009 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4010 sumo_rlc_fini(rdev);
4011 return r;
4012 }
4013 /* set up the cs buffer */
4014 dst_ptr = rdev->rlc.cs_ptr;
4015 reg_list_hdr_blk_index = 0;
4016 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4017 data = upper_32_bits(reg_list_mc_addr);
4018 dst_ptr[reg_list_hdr_blk_index] = data;
4019 reg_list_hdr_blk_index++;
4020 for (i = 0; cs_data[i].section != NULL; i++) {
4021 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4022 reg_num = cs_data[i].section[j].reg_count;
4023 data = reg_list_mc_addr & 0xffffffff;
4024 dst_ptr[reg_list_hdr_blk_index] = data;
4025 reg_list_hdr_blk_index++;
4026
4027 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4028 dst_ptr[reg_list_hdr_blk_index] = data;
4029 reg_list_hdr_blk_index++;
4030
4031 data = 0x08000000 | (reg_num * 4);
4032 dst_ptr[reg_list_hdr_blk_index] = data;
4033 reg_list_hdr_blk_index++;
4034
4035 for (k = 0; k < reg_num; k++) {
4036 data = cs_data[i].section[j].extent[k];
4037 dst_ptr[reg_list_blk_index + k] = data;
4038 }
4039 reg_list_mc_addr += reg_num * 4;
4040 reg_list_blk_index += reg_num;
2948f5e6 4041 }
2948f5e6 4042 }
10b7ca7e 4043 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
2948f5e6 4044
10b7ca7e
AD		 4045 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4046 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4047 }
2948f5e6
AD		 4048
4049 return 0;
4050}
4051
4052static void evergreen_rlc_start(struct radeon_device *rdev)
4053{
8ba10463
AD		 4054 u32 mask = RLC_ENABLE;
4055
4056 if (rdev->flags & RADEON_IS_IGP) {
4057 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
8ba10463
AD		 4058 }
4059
4060 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4061}
4062
4063int evergreen_rlc_resume(struct radeon_device *rdev)
4064{
4065 u32 i;
4066 const __be32 *fw_data;
4067
4068 if (!rdev->rlc_fw)
4069 return -EINVAL;
4070
4071 r600_rlc_stop(rdev);
4072
4073 WREG32(RLC_HB_CNTL, 0);
4074
4075 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4076 if (rdev->family == CHIP_ARUBA) {
4077 u32 always_on_bitmap =
4078 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4079 /* find out the number of active simds */
4080 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4081 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4082 tmp = hweight32(~tmp);
4083 if (tmp == rdev->config.cayman.max_simds_per_se) {
4084 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4085 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4086 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4087 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4088 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4089 }
4090 } else {
4091 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4092 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4093 }
2948f5e6
AD		 4094 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4095 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4096 } else {
4097 WREG32(RLC_HB_BASE, 0);
4098 WREG32(RLC_HB_RPTR, 0);
4099 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4100 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4101 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4102 }
2948f5e6
AD		 4103 WREG32(RLC_MC_CNTL, 0);
4104 WREG32(RLC_UCODE_CNTL, 0);
4105
4106 fw_data = (const __be32 *)rdev->rlc_fw->data;
4107 if (rdev->family >= CHIP_ARUBA) {
4108 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4109 WREG32(RLC_UCODE_ADDR, i);
4110 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4111 }
4112 } else if (rdev->family >= CHIP_CAYMAN) {
4113 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4114 WREG32(RLC_UCODE_ADDR, i);
4115 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4116 }
4117 } else {
4118 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4119 WREG32(RLC_UCODE_ADDR, i);
4120 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4121 }
4122 }
4123 WREG32(RLC_UCODE_ADDR, 0);
4124
4125 evergreen_rlc_start(rdev);
4126
4127 return 0;
4128}
4129
45f9a39b
AD		 4130/* Interrupts */
4131
4132u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4133{
46437057 4134 if (crtc >= rdev->num_crtc)
45f9a39b 4135 return 0;
46437057
AD		 4136 else
4137 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4138}
4139
4140void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4141{
4142 u32 tmp;
4143
1b37078b
AD		 4144 if (rdev->family >= CHIP_CAYMAN) {
4145 cayman_cp_int_cntl_setup(rdev, 0,
4146 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4147 cayman_cp_int_cntl_setup(rdev, 1, 0);
4148 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4149 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4150 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4151 } else
4152 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4153 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4154 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 4155 WREG32(GRBM_INT_CNTL, 0);
4156 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4157 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4158 if (rdev->num_crtc >= 4) {
18007401
AD		 4159 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4160 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4161 }
4162 if (rdev->num_crtc >= 6) {
18007401
AD		 4163 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4164 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4165 }
45f9a39b
AD		 4166
4167 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4168 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4169 if (rdev->num_crtc >= 4) {
18007401
AD		 4170 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4171 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4172 }
4173 if (rdev->num_crtc >= 6) {
18007401
AD		 4174 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4175 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4176 }
45f9a39b 4177
05b3ef69
AD		 4178 /* only one DAC on DCE6 */
4179 if (!ASIC_IS_DCE6(rdev))
4180 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4181 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4182
4183 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4184 WREG32(DC_HPD1_INT_CONTROL, tmp);
4185 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4186 WREG32(DC_HPD2_INT_CONTROL, tmp);
4187 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4188 WREG32(DC_HPD3_INT_CONTROL, tmp);
4189 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4190 WREG32(DC_HPD4_INT_CONTROL, tmp);
4191 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4192 WREG32(DC_HPD5_INT_CONTROL, tmp);
4193 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4194 WREG32(DC_HPD6_INT_CONTROL, tmp);
4195
4196}
4197
4198int evergreen_irq_set(struct radeon_device *rdev)
4199{
4200 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4201 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4202 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4203 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4204 u32 grbm_int_cntl = 0;
6f34be50 4205 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
f122c610 4206 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4207 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4208 u32 thermal_int = 0;
45f9a39b
AD		 4209
4210 if (!rdev->irq.installed) {
fce7d61b 4211 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4212 return -EINVAL;
4213 }
4214 /* don't enable anything if the ih is disabled */
4215 if (!rdev->ih.enabled) {
4216 r600_disable_interrupts(rdev);
4217 /* force the active interrupt state to all disabled */
4218 evergreen_disable_interrupt_state(rdev);
4219 return 0;
4220 }
4221
4222 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4223 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4224 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4225 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4226 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4227 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
d70229f7
AD		 4228 if (rdev->family == CHIP_ARUBA)
4229 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4230 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4231 else
4232 thermal_int = RREG32(CG_THERMAL_INT) &
4233 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4234
f122c610
AD		 4235 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4236 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4237 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4238 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4239 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4240 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4241
233d1ad5
AD		 4242 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4243
1b37078b
AD		 4244 if (rdev->family >= CHIP_CAYMAN) {
4245 /* enable CP interrupts on all rings */
736fc37f 4246 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4247 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4248 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4249 }
736fc37f 4250 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4251 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4252 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4253 }
736fc37f 4254 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4255 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4256 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4257 }
4258 } else {
736fc37f 4259 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4260 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4261 cp_int_cntl |= RB_INT_ENABLE;
4262 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4263 }
45f9a39b 4264 }
1b37078b 4265
233d1ad5
AD		 4266 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4267 DRM_DEBUG("r600_irq_set: sw int dma\n");
4268 dma_cntl |= TRAP_ENABLE;
4269 }
4270
f60cbd11
AD		 4271 if (rdev->family >= CHIP_CAYMAN) {
4272 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4273 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4274 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4275 dma_cntl1 |= TRAP_ENABLE;
4276 }
4277 }
4278
dc50ba7f
AD		 4279 if (rdev->irq.dpm_thermal) {
4280 DRM_DEBUG("dpm thermal\n");
4281 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4282 }
4283
6f34be50 4284 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4285 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4286 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4287 crtc1 |= VBLANK_INT_MASK;
4288 }
6f34be50 4289 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4290 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4291 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4292 crtc2 |= VBLANK_INT_MASK;
4293 }
6f34be50 4294 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4295 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4296 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4297 crtc3 |= VBLANK_INT_MASK;
4298 }
6f34be50 4299 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4300 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4301 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4302 crtc4 |= VBLANK_INT_MASK;
4303 }
6f34be50 4304 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4305 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4306 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4307 crtc5 |= VBLANK_INT_MASK;
4308 }
6f34be50 4309 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4310 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4311 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4312 crtc6 |= VBLANK_INT_MASK;
4313 }
4314 if (rdev->irq.hpd[0]) {
4315 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4316 hpd1 |= DC_HPDx_INT_EN;
4317 }
4318 if (rdev->irq.hpd[1]) {
4319 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4320 hpd2 |= DC_HPDx_INT_EN;
4321 }
4322 if (rdev->irq.hpd[2]) {
4323 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4324 hpd3 |= DC_HPDx_INT_EN;
4325 }
4326 if (rdev->irq.hpd[3]) {
4327 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4328 hpd4 |= DC_HPDx_INT_EN;
4329 }
4330 if (rdev->irq.hpd[4]) {
4331 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4332 hpd5 |= DC_HPDx_INT_EN;
4333 }
4334 if (rdev->irq.hpd[5]) {
4335 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4336 hpd6 |= DC_HPDx_INT_EN;
4337 }
f122c610
AD		 4338 if (rdev->irq.afmt[0]) {
4339 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4340 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4341 }
4342 if (rdev->irq.afmt[1]) {
4343 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4344 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4345 }
4346 if (rdev->irq.afmt[2]) {
4347 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4348 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4349 }
4350 if (rdev->irq.afmt[3]) {
4351 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4352 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4353 }
4354 if (rdev->irq.afmt[4]) {
4355 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4356 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4357 }
4358 if (rdev->irq.afmt[5]) {
4359 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4360 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4361 }
45f9a39b 4362
1b37078b
AD		 4363 if (rdev->family >= CHIP_CAYMAN) {
4364 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4365 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4366 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4367 } else
4368 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4369
4370 WREG32(DMA_CNTL, dma_cntl);
4371
f60cbd11
AD		 4372 if (rdev->family >= CHIP_CAYMAN)
4373 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4374
2031f77c 4375 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4376
4377 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4378 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4379 if (rdev->num_crtc >= 4) {
18007401
AD		 4380 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4381 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4382 }
4383 if (rdev->num_crtc >= 6) {
18007401
AD		 4384 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4385 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4386 }
45f9a39b 4387
6f34be50
AD		 4388 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4389 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
b7eff394
AD		 4390 if (rdev->num_crtc >= 4) {
4391 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4392 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4393 }
4394 if (rdev->num_crtc >= 6) {
4395 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4396 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4397 }
6f34be50 4398
45f9a39b
AD		 4399 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4400 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4401 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4402 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4403 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4404 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4405 if (rdev->family == CHIP_ARUBA)
4406 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4407 else
4408 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4409
f122c610
AD		 4410 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4411 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4412 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4413 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4414 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4415 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4416
bcc1c2a1
AD		 4417 return 0;
4418}
4419
cbdd4501 4420static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4421{
4422 u32 tmp;
4423
6f34be50
AD		 4424 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4425 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4426 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4427 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4428 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4429 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4430 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4431 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4432 if (rdev->num_crtc >= 4) {
4433 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4434 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4435 }
4436 if (rdev->num_crtc >= 6) {
4437 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4438 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4439 }
6f34be50 4440
f122c610
AD		 4441 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4442 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4443 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4444 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4445 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4446 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4447
6f34be50
AD		 4448 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4449 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4450 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4451 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4452 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4453 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4454 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4455 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4456 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4457 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4458 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4459 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4460
b7eff394
AD		 4461 if (rdev->num_crtc >= 4) {
4462 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4463 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4464 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4465 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4466 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4467 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4468 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4469 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4470 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4471 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4472 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4473 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4474 }
4475
4476 if (rdev->num_crtc >= 6) {
4477 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4478 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4479 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4480 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4481 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4482 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4483 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4484 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4485 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4486 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4487 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4488 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4489 }
45f9a39b 4490
6f34be50 4491 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4492 tmp = RREG32(DC_HPD1_INT_CONTROL);
4493 tmp |= DC_HPDx_INT_ACK;
4494 WREG32(DC_HPD1_INT_CONTROL, tmp);
4495 }
6f34be50 4496 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4497 tmp = RREG32(DC_HPD2_INT_CONTROL);
4498 tmp |= DC_HPDx_INT_ACK;
4499 WREG32(DC_HPD2_INT_CONTROL, tmp);
4500 }
6f34be50 4501 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4502 tmp = RREG32(DC_HPD3_INT_CONTROL);
4503 tmp |= DC_HPDx_INT_ACK;
4504 WREG32(DC_HPD3_INT_CONTROL, tmp);
4505 }
6f34be50 4506 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4507 tmp = RREG32(DC_HPD4_INT_CONTROL);
4508 tmp |= DC_HPDx_INT_ACK;
4509 WREG32(DC_HPD4_INT_CONTROL, tmp);
4510 }
6f34be50 4511 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4512 tmp = RREG32(DC_HPD5_INT_CONTROL);
4513 tmp |= DC_HPDx_INT_ACK;
4514 WREG32(DC_HPD5_INT_CONTROL, tmp);
4515 }
6f34be50 4516 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4517 tmp = RREG32(DC_HPD5_INT_CONTROL);
4518 tmp |= DC_HPDx_INT_ACK;
4519 WREG32(DC_HPD6_INT_CONTROL, tmp);
4520 }
f122c610
AD		 4521 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4522 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4523 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4524 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4525 }
4526 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4527 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4528 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4529 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4530 }
4531 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4532 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4533 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4534 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4535 }
4536 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4537 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4538 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4539 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4540 }
4541 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4542 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4543 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4544 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4545 }
4546 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4547 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4548 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4549 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4550 }
45f9a39b
AD		 4551}
4552
1109ca09 4553static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4554{
45f9a39b
AD		 4555 r600_disable_interrupts(rdev);
4556 /* Wait and acknowledge irq */
4557 mdelay(1);
6f34be50 4558 evergreen_irq_ack(rdev);
45f9a39b
AD		 4559 evergreen_disable_interrupt_state(rdev);
4560}
4561
755d819e 4562void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4563{
4564 evergreen_irq_disable(rdev);
4565 r600_rlc_stop(rdev);
4566}
4567
cbdd4501 4568static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4569{
4570 u32 wptr, tmp;
4571
724c80e1 4572 if (rdev->wb.enabled)
204ae24d 4573 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4574 else
4575 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4576
4577 if (wptr & RB_OVERFLOW) {
4578 /* When a ring buffer overflow happen start parsing interrupt
4579 * from the last not overwritten vector (wptr + 16). Hopefully
4580 * this should allow us to catchup.
4581 */
4582 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4583 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4584 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4585 tmp = RREG32(IH_RB_CNTL);
4586 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4587 WREG32(IH_RB_CNTL, tmp);
4588 }
4589 return (wptr & rdev->ih.ptr_mask);
4590}
4591
4592int evergreen_irq_process(struct radeon_device *rdev)
4593{
682f1a54
DA		 4594 u32 wptr;
4595 u32 rptr;
45f9a39b
AD		 4596 u32 src_id, src_data;
4597 u32 ring_index;
45f9a39b 4598 bool queue_hotplug = false;
f122c610 4599 bool queue_hdmi = false;
dc50ba7f 4600 bool queue_thermal = false;
54e2e49c 4601 u32 status, addr;
45f9a39b 4602
682f1a54 4603 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4604 return IRQ_NONE;
4605
682f1a54 4606 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4607
4608restart_ih:
4609 /* is somebody else already processing irqs? */
4610 if (atomic_xchg(&rdev->ih.lock, 1))
4611 return IRQ_NONE;
4612
682f1a54
DA		 4613 rptr = rdev->ih.rptr;
4614 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4615
964f6645
BH		 4616 /* Order reading of wptr vs. reading of IH ring data */
4617 rmb();
4618
45f9a39b 4619 /* display interrupts */
6f34be50 4620 evergreen_irq_ack(rdev);
45f9a39b 4621
45f9a39b
AD		 4622 while (rptr != wptr) {
4623 /* wptr/rptr are in bytes! */
4624 ring_index = rptr / 4;
0f234f5f
AD		 4625 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4626 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4627
4628 switch (src_id) {
4629 case 1: /* D1 vblank/vline */
4630 switch (src_data) {
4631 case 0: /* D1 vblank */
6f34be50 4632 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4633 if (rdev->irq.crtc_vblank_int[0]) {
4634 drm_handle_vblank(rdev->ddev, 0);
4635 rdev->pm.vblank_sync = true;
4636 wake_up(&rdev->irq.vblank_queue);
4637 }
736fc37f 4638 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4639 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4640 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4641 DRM_DEBUG("IH: D1 vblank\n");
4642 }
4643 break;
4644 case 1: /* D1 vline */
6f34be50
AD		 4645 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4646 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4647 DRM_DEBUG("IH: D1 vline\n");
4648 }
4649 break;
4650 default:
4651 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4652 break;
4653 }
4654 break;
4655 case 2: /* D2 vblank/vline */
4656 switch (src_data) {
4657 case 0: /* D2 vblank */
6f34be50 4658 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4659 if (rdev->irq.crtc_vblank_int[1]) {
4660 drm_handle_vblank(rdev->ddev, 1);
4661 rdev->pm.vblank_sync = true;
4662 wake_up(&rdev->irq.vblank_queue);
4663 }
736fc37f 4664 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4665 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4666 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4667 DRM_DEBUG("IH: D2 vblank\n");
4668 }
4669 break;
4670 case 1: /* D2 vline */
6f34be50
AD		 4671 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4672 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4673 DRM_DEBUG("IH: D2 vline\n");
4674 }
4675 break;
4676 default:
4677 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4678 break;
4679 }
4680 break;
4681 case 3: /* D3 vblank/vline */
4682 switch (src_data) {
4683 case 0: /* D3 vblank */
6f34be50
AD		 4684 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4685 if (rdev->irq.crtc_vblank_int[2]) {
4686 drm_handle_vblank(rdev->ddev, 2);
4687 rdev->pm.vblank_sync = true;
4688 wake_up(&rdev->irq.vblank_queue);
4689 }
736fc37f 4690 if (atomic_read(&rdev->irq.pflip[2]))
6f34be50
AD		 4691 radeon_crtc_handle_flip(rdev, 2);
4692 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4693 DRM_DEBUG("IH: D3 vblank\n");
4694 }
4695 break;
4696 case 1: /* D3 vline */
6f34be50
AD		 4697 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4698 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4699 DRM_DEBUG("IH: D3 vline\n");
4700 }
4701 break;
4702 default:
4703 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4704 break;
4705 }
4706 break;
4707 case 4: /* D4 vblank/vline */
4708 switch (src_data) {
4709 case 0: /* D4 vblank */
6f34be50
AD		 4710 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4711 if (rdev->irq.crtc_vblank_int[3]) {
4712 drm_handle_vblank(rdev->ddev, 3);
4713 rdev->pm.vblank_sync = true;
4714 wake_up(&rdev->irq.vblank_queue);
4715 }
736fc37f 4716 if (atomic_read(&rdev->irq.pflip[3]))
6f34be50
AD		 4717 radeon_crtc_handle_flip(rdev, 3);
4718 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4719 DRM_DEBUG("IH: D4 vblank\n");
4720 }
4721 break;
4722 case 1: /* D4 vline */
6f34be50
AD		 4723 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4724 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4725 DRM_DEBUG("IH: D4 vline\n");
4726 }
4727 break;
4728 default:
4729 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4730 break;
4731 }
4732 break;
4733 case 5: /* D5 vblank/vline */
4734 switch (src_data) {
4735 case 0: /* D5 vblank */
6f34be50
AD		 4736 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4737 if (rdev->irq.crtc_vblank_int[4]) {
4738 drm_handle_vblank(rdev->ddev, 4);
4739 rdev->pm.vblank_sync = true;
4740 wake_up(&rdev->irq.vblank_queue);
4741 }
736fc37f 4742 if (atomic_read(&rdev->irq.pflip[4]))
6f34be50
AD		 4743 radeon_crtc_handle_flip(rdev, 4);
4744 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4745 DRM_DEBUG("IH: D5 vblank\n");
4746 }
4747 break;
4748 case 1: /* D5 vline */
6f34be50
AD		 4749 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4750 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4751 DRM_DEBUG("IH: D5 vline\n");
4752 }
4753 break;
4754 default:
4755 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4756 break;
4757 }
4758 break;
4759 case 6: /* D6 vblank/vline */
4760 switch (src_data) {
4761 case 0: /* D6 vblank */
6f34be50
AD		 4762 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4763 if (rdev->irq.crtc_vblank_int[5]) {
4764 drm_handle_vblank(rdev->ddev, 5);
4765 rdev->pm.vblank_sync = true;
4766 wake_up(&rdev->irq.vblank_queue);
4767 }
736fc37f 4768 if (atomic_read(&rdev->irq.pflip[5]))
6f34be50
AD		 4769 radeon_crtc_handle_flip(rdev, 5);
4770 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4771 DRM_DEBUG("IH: D6 vblank\n");
4772 }
4773 break;
4774 case 1: /* D6 vline */
6f34be50
AD		 4775 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4776 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4777 DRM_DEBUG("IH: D6 vline\n");
4778 }
4779 break;
4780 default:
4781 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4782 break;
4783 }
4784 break;
4785 case 42: /* HPD hotplug */
4786 switch (src_data) {
4787 case 0:
6f34be50
AD		 4788 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4789 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4790 queue_hotplug = true;
4791 DRM_DEBUG("IH: HPD1\n");
4792 }
4793 break;
4794 case 1:
6f34be50
AD		 4795 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4796 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4797 queue_hotplug = true;
4798 DRM_DEBUG("IH: HPD2\n");
4799 }
4800 break;
4801 case 2:
6f34be50
AD		 4802 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4803 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4804 queue_hotplug = true;
4805 DRM_DEBUG("IH: HPD3\n");
4806 }
4807 break;
4808 case 3:
6f34be50
AD		 4809 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4810 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 4811 queue_hotplug = true;
4812 DRM_DEBUG("IH: HPD4\n");
4813 }
4814 break;
4815 case 4:
6f34be50
AD		 4816 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4817 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 4818 queue_hotplug = true;
4819 DRM_DEBUG("IH: HPD5\n");
4820 }
4821 break;
4822 case 5:
6f34be50
AD		 4823 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4824 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 4825 queue_hotplug = true;
4826 DRM_DEBUG("IH: HPD6\n");
4827 }
4828 break;
4829 default:
4830 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4831 break;
4832 }
4833 break;
f122c610
AD		 4834 case 44: /* hdmi */
4835 switch (src_data) {
4836 case 0:
4837 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4838 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4839 queue_hdmi = true;
4840 DRM_DEBUG("IH: HDMI0\n");
4841 }
4842 break;
4843 case 1:
4844 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4845 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4846 queue_hdmi = true;
4847 DRM_DEBUG("IH: HDMI1\n");
4848 }
4849 break;
4850 case 2:
4851 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4852 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4853 queue_hdmi = true;
4854 DRM_DEBUG("IH: HDMI2\n");
4855 }
4856 break;
4857 case 3:
4858 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4859 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4860 queue_hdmi = true;
4861 DRM_DEBUG("IH: HDMI3\n");
4862 }
4863 break;
4864 case 4:
4865 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4866 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4867 queue_hdmi = true;
4868 DRM_DEBUG("IH: HDMI4\n");
4869 }
4870 break;
4871 case 5:
4872 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4873 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4874 queue_hdmi = true;
4875 DRM_DEBUG("IH: HDMI5\n");
4876 }
4877 break;
4878 default:
4879 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4880 break;
4881 }
f2ba57b5
CK		 4882 case 124: /* UVD */
4883 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4884 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 4885 break;
ae133a11
CK		 4886 case 146:
4887 case 147:
54e2e49c
AD		 4888 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4889 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 4890 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4891 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
54e2e49c 4892 addr);
ae133a11 4893 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
54e2e49c
AD		 4894 status);
4895 cayman_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 4896 /* reset addr and status */
4897 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4898 break;
45f9a39b
AD		 4899 case 176: /* CP_INT in ring buffer */
4900 case 177: /* CP_INT in IB1 */
4901 case 178: /* CP_INT in IB2 */
4902 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 4903 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 4904 break;
4905 case 181: /* CP EOP event */
4906 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 4907 if (rdev->family >= CHIP_CAYMAN) {
4908 switch (src_data) {
4909 case 0:
4910 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4911 break;
4912 case 1:
4913 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4914 break;
4915 case 2:
4916 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4917 break;
4918 }
4919 } else
4920 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 4921 break;
233d1ad5
AD		 4922 case 224: /* DMA trap event */
4923 DRM_DEBUG("IH: DMA trap\n");
4924 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4925 break;
dc50ba7f
AD		 4926 case 230: /* thermal low to high */
4927 DRM_DEBUG("IH: thermal low to high\n");
4928 rdev->pm.dpm.thermal.high_to_low = false;
4929 queue_thermal = true;
4930 break;
4931 case 231: /* thermal high to low */
4932 DRM_DEBUG("IH: thermal high to low\n");
4933 rdev->pm.dpm.thermal.high_to_low = true;
4934 queue_thermal = true;
4935 break;
2031f77c 4936 case 233: /* GUI IDLE */
303c805c 4937 DRM_DEBUG("IH: GUI idle\n");
2031f77c 4938 break;
f60cbd11
AD		 4939 case 244: /* DMA trap event */
4940 if (rdev->family >= CHIP_CAYMAN) {
4941 DRM_DEBUG("IH: DMA1 trap\n");
4942 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4943 }
4944 break;
45f9a39b
AD		 4945 default:
4946 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4947 break;
4948 }
4949
4950 /* wptr/rptr are in bytes! */
4951 rptr += 16;
4952 rptr &= rdev->ih.ptr_mask;
4953 }
45f9a39b 4954 if (queue_hotplug)
32c87fca 4955 schedule_work(&rdev->hotplug_work);
f122c610
AD		 4956 if (queue_hdmi)
4957 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 4958 if (queue_thermal && rdev->pm.dpm_enabled)
4959 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b
AD		 4960 rdev->ih.rptr = rptr;
4961 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 4962 atomic_set(&rdev->ih.lock, 0);
4963
4964 /* make sure wptr hasn't changed while processing */
4965 wptr = evergreen_get_ih_wptr(rdev);
4966 if (wptr != rptr)
4967 goto restart_ih;
4968
45f9a39b
AD		 4969 return IRQ_HANDLED;
4970}
4971
233d1ad5
AD		 4972/**
4973 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
4974 *
4975 * @rdev: radeon_device pointer
4976 * @fence: radeon fence object
4977 *
4978 * Add a DMA fence packet to the ring to write
4979 * the fence seq number and DMA trap packet to generate
4980 * an interrupt if needed (evergreen-SI).
4981 */
4982void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
4983 struct radeon_fence *fence)
4984{
4985 struct radeon_ring *ring = &rdev->ring[fence->ring];
4986 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4987 /* write the fence */
0fcb6155 4988 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
233d1ad5
AD		 4989 radeon_ring_write(ring, addr & 0xfffffffc);
4990 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
4991 radeon_ring_write(ring, fence->seq);
4992 /* generate an interrupt */
0fcb6155 4993 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
233d1ad5 4994 /* flush HDP */
0fcb6155 4995 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4b681c28 4996 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
233d1ad5
AD		 4997 radeon_ring_write(ring, 1);
4998}
4999
5000/**
5001 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
5002 *
5003 * @rdev: radeon_device pointer
5004 * @ib: IB object to schedule
5005 *
5006 * Schedule an IB in the DMA ring (evergreen).
5007 */
5008void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
5009 struct radeon_ib *ib)
5010{
5011 struct radeon_ring *ring = &rdev->ring[ib->ring];
5012
5013 if (rdev->wb.enabled) {
5014 u32 next_rptr = ring->wptr + 4;
5015 while ((next_rptr & 7) != 5)
5016 next_rptr++;
5017 next_rptr += 3;
0fcb6155 5018 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
233d1ad5
AD		 5019 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5020 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
5021 radeon_ring_write(ring, next_rptr);
5022 }
5023
5024 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
5025 * Pad as necessary with NOPs.
5026 */
5027 while ((ring->wptr & 7) != 5)
0fcb6155
JG		 5028 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5029 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
233d1ad5
AD		 5030 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
5031 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
5032
5033}
5034
5035/**
5036 * evergreen_copy_dma - copy pages using the DMA engine
5037 *
5038 * @rdev: radeon_device pointer
5039 * @src_offset: src GPU address
5040 * @dst_offset: dst GPU address
5041 * @num_gpu_pages: number of GPU pages to xfer
5042 * @fence: radeon fence object
5043 *
5044 * Copy GPU paging using the DMA engine (evergreen-cayman).
5045 * Used by the radeon ttm implementation to move pages if
5046 * registered as the asic copy callback.
5047 */
5048int evergreen_copy_dma(struct radeon_device *rdev,
5049 uint64_t src_offset, uint64_t dst_offset,
5050 unsigned num_gpu_pages,
5051 struct radeon_fence **fence)
5052{
5053 struct radeon_semaphore *sem = NULL;
5054 int ring_index = rdev->asic->copy.dma_ring_index;
5055 struct radeon_ring *ring = &rdev->ring[ring_index];
5056 u32 size_in_dw, cur_size_in_dw;
5057 int i, num_loops;
5058 int r = 0;
5059
5060 r = radeon_semaphore_create(rdev, &sem);
5061 if (r) {
5062 DRM_ERROR("radeon: moving bo (%d).\n", r);
5063 return r;
5064 }
5065
5066 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
5067 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
5068 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5069 if (r) {
5070 DRM_ERROR("radeon: moving bo (%d).\n", r);
5071 radeon_semaphore_free(rdev, &sem, NULL);
5072 return r;
5073 }
5074
5075 if (radeon_fence_need_sync(*fence, ring->idx)) {
5076 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5077 ring->idx);
5078 radeon_fence_note_sync(*fence, ring->idx);
5079 } else {
5080 radeon_semaphore_free(rdev, &sem, NULL);
5081 }
5082
5083 for (i = 0; i < num_loops; i++) {
5084 cur_size_in_dw = size_in_dw;
5085 if (cur_size_in_dw > 0xFFFFF)
5086 cur_size_in_dw = 0xFFFFF;
5087 size_in_dw -= cur_size_in_dw;
0fcb6155 5088 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
233d1ad5
AD		 5089 radeon_ring_write(ring, dst_offset & 0xfffffffc);
5090 radeon_ring_write(ring, src_offset & 0xfffffffc);
5091 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5092 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5093 src_offset += cur_size_in_dw * 4;
5094 dst_offset += cur_size_in_dw * 4;
5095 }
5096
5097 r = radeon_fence_emit(rdev, fence, ring->idx);
5098 if (r) {
5099 radeon_ring_unlock_undo(rdev, ring);
5100 return r;
5101 }
5102
5103 radeon_ring_unlock_commit(rdev, ring);
5104 radeon_semaphore_free(rdev, &sem, *fence);
5105
5106 return r;
5107}
5108
bcc1c2a1
AD		 5109static int evergreen_startup(struct radeon_device *rdev)
5110{
f2ba57b5 5111 struct radeon_ring *ring;
bcc1c2a1
AD		 5112 int r;
5113
9e46a48d 5114 /* enable pcie gen2 link */
cd54033a 5115 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5116 /* enable aspm */
5117 evergreen_program_aspm(rdev);
9e46a48d 5118
6fab3feb
AD		 5119 evergreen_mc_program(rdev);
5120
0af62b01
AD		 5121 if (ASIC_IS_DCE5(rdev)) {
5122 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5123 r = ni_init_microcode(rdev);
5124 if (r) {
5125 DRM_ERROR("Failed to load firmware!\n");
5126 return r;
5127 }
5128 }
755d819e 5129 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5130 if (r) {
0af62b01 5131 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5132 return r;
5133 }
0af62b01
AD		 5134 } else {
5135 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5136 r = r600_init_microcode(rdev);
5137 if (r) {
5138 DRM_ERROR("Failed to load firmware!\n");
5139 return r;
5140 }
5141 }
bcc1c2a1 5142 }
fe251e2f 5143
16cdf04d
AD		 5144 r = r600_vram_scratch_init(rdev);
5145 if (r)
5146 return r;
5147
bcc1c2a1 5148 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5149 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5150 } else {
5151 r = evergreen_pcie_gart_enable(rdev);
5152 if (r)
5153 return r;
5154 }
bcc1c2a1 5155 evergreen_gpu_init(rdev);
bcc1c2a1 5156
2948f5e6
AD		 5157 /* allocate rlc buffers */
5158 if (rdev->flags & RADEON_IS_IGP) {
5159 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
1fd11777
AD		 5160 rdev->rlc.reg_list_size =
5161 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
2948f5e6
AD		 5162 rdev->rlc.cs_data = evergreen_cs_data;
5163 r = sumo_rlc_init(rdev);
5164 if (r) {
5165 DRM_ERROR("Failed to init rlc BOs!\n");
5166 return r;
5167 }
5168 }
5169
724c80e1
AD		 5170 /* allocate wb buffer */
5171 r = radeon_wb_init(rdev);
5172 if (r)
5173 return r;
5174
30eb77f4
JG		 5175 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5176 if (r) {
5177 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5178 return r;
5179 }
5180
233d1ad5
AD		 5181 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5182 if (r) {
5183 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5184 return r;
5185 }
5186
f2ba57b5
CK		 5187 r = rv770_uvd_resume(rdev);
5188 if (!r) {
5189 r = radeon_fence_driver_start_ring(rdev,
5190 R600_RING_TYPE_UVD_INDEX);
5191 if (r)
5192 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5193 }
5194
5195 if (r)
5196 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5197
bcc1c2a1 5198 /* Enable IRQ */
e49f3959
AH		 5199 if (!rdev->irq.installed) {
5200 r = radeon_irq_kms_init(rdev);
5201 if (r)
5202 return r;
5203 }
5204
bcc1c2a1
AD		 5205 r = r600_irq_init(rdev);
5206 if (r) {
5207 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5208 radeon_irq_kms_fini(rdev);
5209 return r;
5210 }
45f9a39b 5211 evergreen_irq_set(rdev);
bcc1c2a1 5212
f2ba57b5 5213 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5214 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a
AD		 5215 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5216 0, 0xfffff, RADEON_CP_PACKET2);
bcc1c2a1
AD		 5217 if (r)
5218 return r;
233d1ad5
AD		 5219
5220 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5221 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5222 DMA_RB_RPTR, DMA_RB_WPTR,
0fcb6155 5223 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5224 if (r)
5225 return r;
5226
bcc1c2a1
AD		 5227 r = evergreen_cp_load_microcode(rdev);
5228 if (r)
5229 return r;
fe251e2f 5230 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5231 if (r)
5232 return r;
5233 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5234 if (r)
5235 return r;
fe251e2f 5236
f2ba57b5
CK		 5237 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5238 if (ring->ring_size) {
5239 r = radeon_ring_init(rdev, ring, ring->ring_size,
5240 R600_WB_UVD_RPTR_OFFSET,
5241 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5242 0, 0xfffff, RADEON_CP_PACKET2);
5243 if (!r)
5244 r = r600_uvd_init(rdev);
5245
5246 if (r)
5247 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5248 }
5249
2898c348
CK		 5250 r = radeon_ib_pool_init(rdev);
5251 if (r) {
5252 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5253 return r;
2898c348 5254 }
b15ba512 5255
69d2ae57
RM		 5256 r = r600_audio_init(rdev);
5257 if (r) {
5258 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5259 return r;
5260 }
5261
bcc1c2a1
AD		 5262 return 0;
5263}
5264
5265int evergreen_resume(struct radeon_device *rdev)
5266{
5267 int r;
5268
86f5c9ed
AD		 5269 /* reset the asic, the gfx blocks are often in a bad state
5270 * after the driver is unloaded or after a resume
5271 */
5272 if (radeon_asic_reset(rdev))
5273 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5274 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5275 * posting will perform necessary task to bring back GPU into good
5276 * shape.
5277 */
5278 /* post card */
5279 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5280
d4788db3
AD		 5281 /* init golden registers */
5282 evergreen_init_golden_registers(rdev);
5283
b15ba512 5284 rdev->accel_working = true;
bcc1c2a1
AD		 5285 r = evergreen_startup(rdev);
5286 if (r) {
755d819e 5287 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5288 rdev->accel_working = false;
bcc1c2a1
AD		 5289 return r;
5290 }
fe251e2f 5291
bcc1c2a1
AD		 5292 return r;
5293
5294}
5295
5296int evergreen_suspend(struct radeon_device *rdev)
5297{
69d2ae57 5298 r600_audio_fini(rdev);
2858c00d 5299 r600_uvd_stop(rdev);
f2ba57b5 5300 radeon_uvd_suspend(rdev);
bcc1c2a1 5301 r700_cp_stop(rdev);
233d1ad5 5302 r600_dma_stop(rdev);
45f9a39b 5303 evergreen_irq_suspend(rdev);
724c80e1 5304 radeon_wb_disable(rdev);
bcc1c2a1 5305 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5306
5307 return 0;
5308}
5309
bcc1c2a1
AD		 5310/* Plan is to move initialization in that function and use
5311 * helper function so that radeon_device_init pretty much
5312 * do nothing more than calling asic specific function. This
5313 * should also allow to remove a bunch of callback function
5314 * like vram_info.
5315 */
5316int evergreen_init(struct radeon_device *rdev)
5317{
5318 int r;
5319
bcc1c2a1
AD		 5320 /* Read BIOS */
5321 if (!radeon_get_bios(rdev)) {
5322 if (ASIC_IS_AVIVO(rdev))
5323 return -EINVAL;
5324 }
5325 /* Must be an ATOMBIOS */
5326 if (!rdev->is_atom_bios) {
755d819e 5327 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5328 return -EINVAL;
5329 }
5330 r = radeon_atombios_init(rdev);
5331 if (r)
5332 return r;
86f5c9ed
AD		 5333 /* reset the asic, the gfx blocks are often in a bad state
5334 * after the driver is unloaded or after a resume
5335 */
5336 if (radeon_asic_reset(rdev))
5337 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5338 /* Post card if necessary */
fd909c37 5339 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5340 if (!rdev->bios) {
5341 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5342 return -EINVAL;
5343 }
5344 DRM_INFO("GPU not posted. posting now...\n");
5345 atom_asic_init(rdev->mode_info.atom_context);
5346 }
d4788db3
AD		 5347 /* init golden registers */
5348 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5349 /* Initialize scratch registers */
5350 r600_scratch_init(rdev);
5351 /* Initialize surface registers */
5352 radeon_surface_init(rdev);
5353 /* Initialize clocks */
5354 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5355 /* Fence driver */
5356 r = radeon_fence_driver_init(rdev);
5357 if (r)
5358 return r;
d594e46a
JG		 5359 /* initialize AGP */
5360 if (rdev->flags & RADEON_IS_AGP) {
5361 r = radeon_agp_init(rdev);
5362 if (r)
5363 radeon_agp_disable(rdev);
5364 }
5365 /* initialize memory controller */
bcc1c2a1
AD		 5366 r = evergreen_mc_init(rdev);
5367 if (r)
5368 return r;
5369 /* Memory manager */
5370 r = radeon_bo_init(rdev);
5371 if (r)
5372 return r;
45f9a39b 5373
e32eb50d
CK		 5374 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5375 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5376
233d1ad5
AD		 5377 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5378 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5379
f2ba57b5
CK		 5380 r = radeon_uvd_init(rdev);
5381 if (!r) {
5382 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5383 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5384 4096);
5385 }
5386
bcc1c2a1
AD		 5387 rdev->ih.ring_obj = NULL;
5388 r600_ih_ring_init(rdev, 64 * 1024);
5389
5390 r = r600_pcie_gart_init(rdev);
5391 if (r)
5392 return r;
0fcdb61e 5393
148a03bc 5394 rdev->accel_working = true;
bcc1c2a1
AD		 5395 r = evergreen_startup(rdev);
5396 if (r) {
fe251e2f
AD		 5397 dev_err(rdev->dev, "disabling GPU acceleration\n");
5398 r700_cp_fini(rdev);
233d1ad5 5399 r600_dma_fini(rdev);
fe251e2f 5400 r600_irq_fini(rdev);
2948f5e6
AD		 5401 if (rdev->flags & RADEON_IS_IGP)
5402 sumo_rlc_fini(rdev);
724c80e1 5403 radeon_wb_fini(rdev);
2898c348 5404 radeon_ib_pool_fini(rdev);
fe251e2f 5405 radeon_irq_kms_fini(rdev);
0fcdb61e 5406 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5407 rdev->accel_working = false;
5408 }
77e00f2e
AD		 5409
5410 /* Don't start up if the MC ucode is missing on BTC parts.
5411 * The default clocks and voltages before the MC ucode
5412 * is loaded are not suffient for advanced operations.
5413 */
5414 if (ASIC_IS_DCE5(rdev)) {
5415 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5416 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5417 return -EINVAL;
5418 }
5419 }
5420
bcc1c2a1
AD		 5421 return 0;
5422}
5423
5424void evergreen_fini(struct radeon_device *rdev)
5425{
69d2ae57 5426 r600_audio_fini(rdev);
45f9a39b 5427 r700_cp_fini(rdev);
233d1ad5 5428 r600_dma_fini(rdev);
bcc1c2a1 5429 r600_irq_fini(rdev);
2948f5e6
AD		 5430 if (rdev->flags & RADEON_IS_IGP)
5431 sumo_rlc_fini(rdev);
724c80e1 5432 radeon_wb_fini(rdev);
2898c348 5433 radeon_ib_pool_fini(rdev);
bcc1c2a1 5434 radeon_irq_kms_fini(rdev);
bcc1c2a1 5435 evergreen_pcie_gart_fini(rdev);
2858c00d 5436 r600_uvd_stop(rdev);
f2ba57b5 5437 radeon_uvd_fini(rdev);
16cdf04d 5438 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5439 radeon_gem_fini(rdev);
5440 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5441 radeon_agp_fini(rdev);
5442 radeon_bo_fini(rdev);
5443 radeon_atombios_fini(rdev);
5444 kfree(rdev->bios);
5445 rdev->bios = NULL;
bcc1c2a1 5446}
9e46a48d 5447
b07759bf 5448void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5449{
7e0e4196 5450 u32 link_width_cntl, speed_cntl;
9e46a48d 5451
d42dd579
AD		 5452 if (radeon_pcie_gen2 == 0)
5453 return;
5454
9e46a48d
AD		 5455 if (rdev->flags & RADEON_IS_IGP)
5456 return;
5457
5458 if (!(rdev->flags & RADEON_IS_PCIE))
5459 return;
5460
5461 /* x2 cards have a special sequence */
5462 if (ASIC_IS_X2(rdev))
5463 return;
5464
7e0e4196
KSS		 5465 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5466 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5467 return;
5468
492d2b61 5469 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5470 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5471 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5472 return;
5473 }
5474
197bbb3d
DA		 5475 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5476
9e46a48d
AD		 5477 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5478 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5479
492d2b61 5480 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5481 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5482 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5483
492d2b61 5484 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5485 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5486 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5487
492d2b61 5488 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5489 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5490 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5491
492d2b61 5492 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5493 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5494 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5495
492d2b61 5496 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5497 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5498 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5499
5500 } else {
492d2b61 5501 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5502 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5503 if (1)
5504 link_width_cntl |= LC_UPCONFIGURE_DIS;
5505 else
5506 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5507 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5508 }
5509}
f52382d7
AD		 5510
5511void evergreen_program_aspm(struct radeon_device *rdev)
5512{
5513 u32 data, orig;
5514 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5515 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5516 /* fusion_platform = true
5517 * if the system is a fusion system
5518 * (APU or DGPU in a fusion system).
5519 * todo: check if the system is a fusion platform.
5520 */
5521 bool fusion_platform = false;
5522
1294d4a3
AD		 5523 if (radeon_aspm == 0)
5524 return;
5525
f52382d7
AD		 5526 if (!(rdev->flags & RADEON_IS_PCIE))
5527 return;
5528
5529 switch (rdev->family) {
5530 case CHIP_CYPRESS:
5531 case CHIP_HEMLOCK:
5532 case CHIP_JUNIPER:
5533 case CHIP_REDWOOD:
5534 case CHIP_CEDAR:
5535 case CHIP_SUMO:
5536 case CHIP_SUMO2:
5537 case CHIP_PALM:
5538 case CHIP_ARUBA:
5539 disable_l0s = true;
5540 break;
5541 default:
5542 disable_l0s = false;
5543 break;
5544 }
5545
5546 if (rdev->flags & RADEON_IS_IGP)
5547 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5548
5549 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5550 if (fusion_platform)
5551 data &= ~MULTI_PIF;
5552 else
5553 data |= MULTI_PIF;
5554 if (data != orig)
5555 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5556
5557 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5558 if (fusion_platform)
5559 data &= ~MULTI_PIF;
5560 else
5561 data |= MULTI_PIF;
5562 if (data != orig)
5563 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5564
5565 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5566 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5567 if (!disable_l0s) {
5568 if (rdev->family >= CHIP_BARTS)
5569 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5570 else
5571 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5572 }
5573
5574 if (!disable_l1) {
5575 if (rdev->family >= CHIP_BARTS)
5576 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5577 else
5578 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5579
5580 if (!disable_plloff_in_l1) {
5581 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5582 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5583 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5584 if (data != orig)
5585 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5586
5587 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5588 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5589 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5590 if (data != orig)
5591 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5592
5593 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5594 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5595 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5596 if (data != orig)
5597 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5598
5599 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5600 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5601 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5602 if (data != orig)
5603 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5604
5605 if (rdev->family >= CHIP_BARTS) {
5606 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5607 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5608 data |= PLL_RAMP_UP_TIME_0(4);
5609 if (data != orig)
5610 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5611
5612 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5613 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5614 data |= PLL_RAMP_UP_TIME_1(4);
5615 if (data != orig)
5616 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5617
5618 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5619 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5620 data |= PLL_RAMP_UP_TIME_0(4);
5621 if (data != orig)
5622 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5623
5624 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5625 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5626 data |= PLL_RAMP_UP_TIME_1(4);
5627 if (data != orig)
5628 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5629 }
5630
5631 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5632 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5633 data |= LC_DYN_LANES_PWR_STATE(3);
5634 if (data != orig)
5635 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5636
5637 if (rdev->family >= CHIP_BARTS) {
5638 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5639 data &= ~LS2_EXIT_TIME_MASK;
5640 data |= LS2_EXIT_TIME(1);
5641 if (data != orig)
5642 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5643
5644 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5645 data &= ~LS2_EXIT_TIME_MASK;
5646 data |= LS2_EXIT_TIME(1);
5647 if (data != orig)
5648 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5649 }
5650 }
5651 }
5652
5653 /* evergreen parts only */
5654 if (rdev->family < CHIP_BARTS)
5655 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5656
5657 if (pcie_lc_cntl != pcie_lc_cntl_old)
5658 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5659}
Linux 6.x block layer and io_uring tree(s)