2 * Copyright 2010 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
36 #include "radeon_ucode.h"
38 static const u32 crtc_offsets[6] =
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
48 #include "clearstate_evergreen.h"
50 static const u32 sumo_rlc_save_restore_register_list[] =
135 static void evergreen_gpu_init(struct radeon_device *rdev);
136 void evergreen_fini(struct radeon_device *rdev);
137 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
138 void evergreen_program_aspm(struct radeon_device *rdev);
139 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
141 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
143 void cik_init_cp_pg_table(struct radeon_device *rdev);
145 extern u32 si_get_csb_size(struct radeon_device *rdev);
146 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
147 extern u32 cik_get_csb_size(struct radeon_device *rdev);
148 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
149 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
151 static const u32 evergreen_golden_registers[] =
153 0x3f90, 0xffff0000, 0xff000000,
154 0x9148, 0xffff0000, 0xff000000,
155 0x3f94, 0xffff0000, 0xff000000,
156 0x914c, 0xffff0000, 0xff000000,
157 0x9b7c, 0xffffffff, 0x00000000,
158 0x8a14, 0xffffffff, 0x00000007,
159 0x8b10, 0xffffffff, 0x00000000,
160 0x960c, 0xffffffff, 0x54763210,
161 0x88c4, 0xffffffff, 0x000000c2,
162 0x88d4, 0xffffffff, 0x00000010,
163 0x8974, 0xffffffff, 0x00000000,
164 0xc78, 0x00000080, 0x00000080,
165 0x5eb4, 0xffffffff, 0x00000002,
166 0x5e78, 0xffffffff, 0x001000f0,
167 0x6104, 0x01000300, 0x00000000,
168 0x5bc0, 0x00300000, 0x00000000,
169 0x7030, 0xffffffff, 0x00000011,
170 0x7c30, 0xffffffff, 0x00000011,
171 0x10830, 0xffffffff, 0x00000011,
172 0x11430, 0xffffffff, 0x00000011,
173 0x12030, 0xffffffff, 0x00000011,
174 0x12c30, 0xffffffff, 0x00000011,
175 0xd02c, 0xffffffff, 0x08421000,
176 0x240c, 0xffffffff, 0x00000380,
177 0x8b24, 0xffffffff, 0x00ff0fff,
178 0x28a4c, 0x06000000, 0x06000000,
179 0x10c, 0x00000001, 0x00000001,
180 0x8d00, 0xffffffff, 0x100e4848,
181 0x8d04, 0xffffffff, 0x00164745,
182 0x8c00, 0xffffffff, 0xe4000003,
183 0x8c04, 0xffffffff, 0x40600060,
184 0x8c08, 0xffffffff, 0x001c001c,
185 0x8cf0, 0xffffffff, 0x08e00620,
186 0x8c20, 0xffffffff, 0x00800080,
187 0x8c24, 0xffffffff, 0x00800080,
188 0x8c18, 0xffffffff, 0x20202078,
189 0x8c1c, 0xffffffff, 0x00001010,
190 0x28350, 0xffffffff, 0x00000000,
191 0xa008, 0xffffffff, 0x00010000,
192 0x5c4, 0xffffffff, 0x00000001,
193 0x9508, 0xffffffff, 0x00000002,
194 0x913c, 0x0000000f, 0x0000000a
197 static const u32 evergreen_golden_registers2[] =
199 0x2f4c, 0xffffffff, 0x00000000,
200 0x54f4, 0xffffffff, 0x00000000,
201 0x54f0, 0xffffffff, 0x00000000,
202 0x5498, 0xffffffff, 0x00000000,
203 0x549c, 0xffffffff, 0x00000000,
204 0x5494, 0xffffffff, 0x00000000,
205 0x53cc, 0xffffffff, 0x00000000,
206 0x53c8, 0xffffffff, 0x00000000,
207 0x53c4, 0xffffffff, 0x00000000,
208 0x53c0, 0xffffffff, 0x00000000,
209 0x53bc, 0xffffffff, 0x00000000,
210 0x53b8, 0xffffffff, 0x00000000,
211 0x53b4, 0xffffffff, 0x00000000,
212 0x53b0, 0xffffffff, 0x00000000
215 static const u32 cypress_mgcg_init[] =
217 0x802c, 0xffffffff, 0xc0000000,
218 0x5448, 0xffffffff, 0x00000100,
219 0x55e4, 0xffffffff, 0x00000100,
220 0x160c, 0xffffffff, 0x00000100,
221 0x5644, 0xffffffff, 0x00000100,
222 0xc164, 0xffffffff, 0x00000100,
223 0x8a18, 0xffffffff, 0x00000100,
224 0x897c, 0xffffffff, 0x06000100,
225 0x8b28, 0xffffffff, 0x00000100,
226 0x9144, 0xffffffff, 0x00000100,
227 0x9a60, 0xffffffff, 0x00000100,
228 0x9868, 0xffffffff, 0x00000100,
229 0x8d58, 0xffffffff, 0x00000100,
230 0x9510, 0xffffffff, 0x00000100,
231 0x949c, 0xffffffff, 0x00000100,
232 0x9654, 0xffffffff, 0x00000100,
233 0x9030, 0xffffffff, 0x00000100,
234 0x9034, 0xffffffff, 0x00000100,
235 0x9038, 0xffffffff, 0x00000100,
236 0x903c, 0xffffffff, 0x00000100,
237 0x9040, 0xffffffff, 0x00000100,
238 0xa200, 0xffffffff, 0x00000100,
239 0xa204, 0xffffffff, 0x00000100,
240 0xa208, 0xffffffff, 0x00000100,
241 0xa20c, 0xffffffff, 0x00000100,
242 0x971c, 0xffffffff, 0x00000100,
243 0x977c, 0xffffffff, 0x00000100,
244 0x3f80, 0xffffffff, 0x00000100,
245 0xa210, 0xffffffff, 0x00000100,
246 0xa214, 0xffffffff, 0x00000100,
247 0x4d8, 0xffffffff, 0x00000100,
248 0x9784, 0xffffffff, 0x00000100,
249 0x9698, 0xffffffff, 0x00000100,
250 0x4d4, 0xffffffff, 0x00000200,
251 0x30cc, 0xffffffff, 0x00000100,
252 0xd0c0, 0xffffffff, 0xff000100,
253 0x802c, 0xffffffff, 0x40000000,
254 0x915c, 0xffffffff, 0x00010000,
255 0x9160, 0xffffffff, 0x00030002,
256 0x9178, 0xffffffff, 0x00070000,
257 0x917c, 0xffffffff, 0x00030002,
258 0x9180, 0xffffffff, 0x00050004,
259 0x918c, 0xffffffff, 0x00010006,
260 0x9190, 0xffffffff, 0x00090008,
261 0x9194, 0xffffffff, 0x00070000,
262 0x9198, 0xffffffff, 0x00030002,
263 0x919c, 0xffffffff, 0x00050004,
264 0x91a8, 0xffffffff, 0x00010006,
265 0x91ac, 0xffffffff, 0x00090008,
266 0x91b0, 0xffffffff, 0x00070000,
267 0x91b4, 0xffffffff, 0x00030002,
268 0x91b8, 0xffffffff, 0x00050004,
269 0x91c4, 0xffffffff, 0x00010006,
270 0x91c8, 0xffffffff, 0x00090008,
271 0x91cc, 0xffffffff, 0x00070000,
272 0x91d0, 0xffffffff, 0x00030002,
273 0x91d4, 0xffffffff, 0x00050004,
274 0x91e0, 0xffffffff, 0x00010006,
275 0x91e4, 0xffffffff, 0x00090008,
276 0x91e8, 0xffffffff, 0x00000000,
277 0x91ec, 0xffffffff, 0x00070000,
278 0x91f0, 0xffffffff, 0x00030002,
279 0x91f4, 0xffffffff, 0x00050004,
280 0x9200, 0xffffffff, 0x00010006,
281 0x9204, 0xffffffff, 0x00090008,
282 0x9208, 0xffffffff, 0x00070000,
283 0x920c, 0xffffffff, 0x00030002,
284 0x9210, 0xffffffff, 0x00050004,
285 0x921c, 0xffffffff, 0x00010006,
286 0x9220, 0xffffffff, 0x00090008,
287 0x9224, 0xffffffff, 0x00070000,
288 0x9228, 0xffffffff, 0x00030002,
289 0x922c, 0xffffffff, 0x00050004,
290 0x9238, 0xffffffff, 0x00010006,
291 0x923c, 0xffffffff, 0x00090008,
292 0x9240, 0xffffffff, 0x00070000,
293 0x9244, 0xffffffff, 0x00030002,
294 0x9248, 0xffffffff, 0x00050004,
295 0x9254, 0xffffffff, 0x00010006,
296 0x9258, 0xffffffff, 0x00090008,
297 0x925c, 0xffffffff, 0x00070000,
298 0x9260, 0xffffffff, 0x00030002,
299 0x9264, 0xffffffff, 0x00050004,
300 0x9270, 0xffffffff, 0x00010006,
301 0x9274, 0xffffffff, 0x00090008,
302 0x9278, 0xffffffff, 0x00070000,
303 0x927c, 0xffffffff, 0x00030002,
304 0x9280, 0xffffffff, 0x00050004,
305 0x928c, 0xffffffff, 0x00010006,
306 0x9290, 0xffffffff, 0x00090008,
307 0x9294, 0xffffffff, 0x00000000,
308 0x929c, 0xffffffff, 0x00000001,
309 0x802c, 0xffffffff, 0x40010000,
310 0x915c, 0xffffffff, 0x00010000,
311 0x9160, 0xffffffff, 0x00030002,
312 0x9178, 0xffffffff, 0x00070000,
313 0x917c, 0xffffffff, 0x00030002,
314 0x9180, 0xffffffff, 0x00050004,
315 0x918c, 0xffffffff, 0x00010006,
316 0x9190, 0xffffffff, 0x00090008,
317 0x9194, 0xffffffff, 0x00070000,
318 0x9198, 0xffffffff, 0x00030002,
319 0x919c, 0xffffffff, 0x00050004,
320 0x91a8, 0xffffffff, 0x00010006,
321 0x91ac, 0xffffffff, 0x00090008,
322 0x91b0, 0xffffffff, 0x00070000,
323 0x91b4, 0xffffffff, 0x00030002,
324 0x91b8, 0xffffffff, 0x00050004,
325 0x91c4, 0xffffffff, 0x00010006,
326 0x91c8, 0xffffffff, 0x00090008,
327 0x91cc, 0xffffffff, 0x00070000,
328 0x91d0, 0xffffffff, 0x00030002,
329 0x91d4, 0xffffffff, 0x00050004,
330 0x91e0, 0xffffffff, 0x00010006,
331 0x91e4, 0xffffffff, 0x00090008,
332 0x91e8, 0xffffffff, 0x00000000,
333 0x91ec, 0xffffffff, 0x00070000,
334 0x91f0, 0xffffffff, 0x00030002,
335 0x91f4, 0xffffffff, 0x00050004,
336 0x9200, 0xffffffff, 0x00010006,
337 0x9204, 0xffffffff, 0x00090008,
338 0x9208, 0xffffffff, 0x00070000,
339 0x920c, 0xffffffff, 0x00030002,
340 0x9210, 0xffffffff, 0x00050004,
341 0x921c, 0xffffffff, 0x00010006,
342 0x9220, 0xffffffff, 0x00090008,
343 0x9224, 0xffffffff, 0x00070000,
344 0x9228, 0xffffffff, 0x00030002,
345 0x922c, 0xffffffff, 0x00050004,
346 0x9238, 0xffffffff, 0x00010006,
347 0x923c, 0xffffffff, 0x00090008,
348 0x9240, 0xffffffff, 0x00070000,
349 0x9244, 0xffffffff, 0x00030002,
350 0x9248, 0xffffffff, 0x00050004,
351 0x9254, 0xffffffff, 0x00010006,
352 0x9258, 0xffffffff, 0x00090008,
353 0x925c, 0xffffffff, 0x00070000,
354 0x9260, 0xffffffff, 0x00030002,
355 0x9264, 0xffffffff, 0x00050004,
356 0x9270, 0xffffffff, 0x00010006,
357 0x9274, 0xffffffff, 0x00090008,
358 0x9278, 0xffffffff, 0x00070000,
359 0x927c, 0xffffffff, 0x00030002,
360 0x9280, 0xffffffff, 0x00050004,
361 0x928c, 0xffffffff, 0x00010006,
362 0x9290, 0xffffffff, 0x00090008,
363 0x9294, 0xffffffff, 0x00000000,
364 0x929c, 0xffffffff, 0x00000001,
365 0x802c, 0xffffffff, 0xc0000000
368 static const u32 redwood_mgcg_init[] =
370 0x802c, 0xffffffff, 0xc0000000,
371 0x5448, 0xffffffff, 0x00000100,
372 0x55e4, 0xffffffff, 0x00000100,
373 0x160c, 0xffffffff, 0x00000100,
374 0x5644, 0xffffffff, 0x00000100,
375 0xc164, 0xffffffff, 0x00000100,
376 0x8a18, 0xffffffff, 0x00000100,
377 0x897c, 0xffffffff, 0x06000100,
378 0x8b28, 0xffffffff, 0x00000100,
379 0x9144, 0xffffffff, 0x00000100,
380 0x9a60, 0xffffffff, 0x00000100,
381 0x9868, 0xffffffff, 0x00000100,
382 0x8d58, 0xffffffff, 0x00000100,
383 0x9510, 0xffffffff, 0x00000100,
384 0x949c, 0xffffffff, 0x00000100,
385 0x9654, 0xffffffff, 0x00000100,
386 0x9030, 0xffffffff, 0x00000100,
387 0x9034, 0xffffffff, 0x00000100,
388 0x9038, 0xffffffff, 0x00000100,
389 0x903c, 0xffffffff, 0x00000100,
390 0x9040, 0xffffffff, 0x00000100,
391 0xa200, 0xffffffff, 0x00000100,
392 0xa204, 0xffffffff, 0x00000100,
393 0xa208, 0xffffffff, 0x00000100,
394 0xa20c, 0xffffffff, 0x00000100,
395 0x971c, 0xffffffff, 0x00000100,
396 0x977c, 0xffffffff, 0x00000100,
397 0x3f80, 0xffffffff, 0x00000100,
398 0xa210, 0xffffffff, 0x00000100,
399 0xa214, 0xffffffff, 0x00000100,
400 0x4d8, 0xffffffff, 0x00000100,
401 0x9784, 0xffffffff, 0x00000100,
402 0x9698, 0xffffffff, 0x00000100,
403 0x4d4, 0xffffffff, 0x00000200,
404 0x30cc, 0xffffffff, 0x00000100,
405 0xd0c0, 0xffffffff, 0xff000100,
406 0x802c, 0xffffffff, 0x40000000,
407 0x915c, 0xffffffff, 0x00010000,
408 0x9160, 0xffffffff, 0x00030002,
409 0x9178, 0xffffffff, 0x00070000,
410 0x917c, 0xffffffff, 0x00030002,
411 0x9180, 0xffffffff, 0x00050004,
412 0x918c, 0xffffffff, 0x00010006,
413 0x9190, 0xffffffff, 0x00090008,
414 0x9194, 0xffffffff, 0x00070000,
415 0x9198, 0xffffffff, 0x00030002,
416 0x919c, 0xffffffff, 0x00050004,
417 0x91a8, 0xffffffff, 0x00010006,
418 0x91ac, 0xffffffff, 0x00090008,
419 0x91b0, 0xffffffff, 0x00070000,
420 0x91b4, 0xffffffff, 0x00030002,
421 0x91b8, 0xffffffff, 0x00050004,
422 0x91c4, 0xffffffff, 0x00010006,
423 0x91c8, 0xffffffff, 0x00090008,
424 0x91cc, 0xffffffff, 0x00070000,
425 0x91d0, 0xffffffff, 0x00030002,
426 0x91d4, 0xffffffff, 0x00050004,
427 0x91e0, 0xffffffff, 0x00010006,
428 0x91e4, 0xffffffff, 0x00090008,
429 0x91e8, 0xffffffff, 0x00000000,
430 0x91ec, 0xffffffff, 0x00070000,
431 0x91f0, 0xffffffff, 0x00030002,
432 0x91f4, 0xffffffff, 0x00050004,
433 0x9200, 0xffffffff, 0x00010006,
434 0x9204, 0xffffffff, 0x00090008,
435 0x9294, 0xffffffff, 0x00000000,
436 0x929c, 0xffffffff, 0x00000001,
437 0x802c, 0xffffffff, 0xc0000000
440 static const u32 cedar_golden_registers[] =
442 0x3f90, 0xffff0000, 0xff000000,
443 0x9148, 0xffff0000, 0xff000000,
444 0x3f94, 0xffff0000, 0xff000000,
445 0x914c, 0xffff0000, 0xff000000,
446 0x9b7c, 0xffffffff, 0x00000000,
447 0x8a14, 0xffffffff, 0x00000007,
448 0x8b10, 0xffffffff, 0x00000000,
449 0x960c, 0xffffffff, 0x54763210,
450 0x88c4, 0xffffffff, 0x000000c2,
451 0x88d4, 0xffffffff, 0x00000000,
452 0x8974, 0xffffffff, 0x00000000,
453 0xc78, 0x00000080, 0x00000080,
454 0x5eb4, 0xffffffff, 0x00000002,
455 0x5e78, 0xffffffff, 0x001000f0,
456 0x6104, 0x01000300, 0x00000000,
457 0x5bc0, 0x00300000, 0x00000000,
458 0x7030, 0xffffffff, 0x00000011,
459 0x7c30, 0xffffffff, 0x00000011,
460 0x10830, 0xffffffff, 0x00000011,
461 0x11430, 0xffffffff, 0x00000011,
462 0xd02c, 0xffffffff, 0x08421000,
463 0x240c, 0xffffffff, 0x00000380,
464 0x8b24, 0xffffffff, 0x00ff0fff,
465 0x28a4c, 0x06000000, 0x06000000,
466 0x10c, 0x00000001, 0x00000001,
467 0x8d00, 0xffffffff, 0x100e4848,
468 0x8d04, 0xffffffff, 0x00164745,
469 0x8c00, 0xffffffff, 0xe4000003,
470 0x8c04, 0xffffffff, 0x40600060,
471 0x8c08, 0xffffffff, 0x001c001c,
472 0x8cf0, 0xffffffff, 0x08e00410,
473 0x8c20, 0xffffffff, 0x00800080,
474 0x8c24, 0xffffffff, 0x00800080,
475 0x8c18, 0xffffffff, 0x20202078,
476 0x8c1c, 0xffffffff, 0x00001010,
477 0x28350, 0xffffffff, 0x00000000,
478 0xa008, 0xffffffff, 0x00010000,
479 0x5c4, 0xffffffff, 0x00000001,
480 0x9508, 0xffffffff, 0x00000002
483 static const u32 cedar_mgcg_init[] =
485 0x802c, 0xffffffff, 0xc0000000,
486 0x5448, 0xffffffff, 0x00000100,
487 0x55e4, 0xffffffff, 0x00000100,
488 0x160c, 0xffffffff, 0x00000100,
489 0x5644, 0xffffffff, 0x00000100,
490 0xc164, 0xffffffff, 0x00000100,
491 0x8a18, 0xffffffff, 0x00000100,
492 0x897c, 0xffffffff, 0x06000100,
493 0x8b28, 0xffffffff, 0x00000100,
494 0x9144, 0xffffffff, 0x00000100,
495 0x9a60, 0xffffffff, 0x00000100,
496 0x9868, 0xffffffff, 0x00000100,
497 0x8d58, 0xffffffff, 0x00000100,
498 0x9510, 0xffffffff, 0x00000100,
499 0x949c, 0xffffffff, 0x00000100,
500 0x9654, 0xffffffff, 0x00000100,
501 0x9030, 0xffffffff, 0x00000100,
502 0x9034, 0xffffffff, 0x00000100,
503 0x9038, 0xffffffff, 0x00000100,
504 0x903c, 0xffffffff, 0x00000100,
505 0x9040, 0xffffffff, 0x00000100,
506 0xa200, 0xffffffff, 0x00000100,
507 0xa204, 0xffffffff, 0x00000100,
508 0xa208, 0xffffffff, 0x00000100,
509 0xa20c, 0xffffffff, 0x00000100,
510 0x971c, 0xffffffff, 0x00000100,
511 0x977c, 0xffffffff, 0x00000100,
512 0x3f80, 0xffffffff, 0x00000100,
513 0xa210, 0xffffffff, 0x00000100,
514 0xa214, 0xffffffff, 0x00000100,
515 0x4d8, 0xffffffff, 0x00000100,
516 0x9784, 0xffffffff, 0x00000100,
517 0x9698, 0xffffffff, 0x00000100,
518 0x4d4, 0xffffffff, 0x00000200,
519 0x30cc, 0xffffffff, 0x00000100,
520 0xd0c0, 0xffffffff, 0xff000100,
521 0x802c, 0xffffffff, 0x40000000,
522 0x915c, 0xffffffff, 0x00010000,
523 0x9178, 0xffffffff, 0x00050000,
524 0x917c, 0xffffffff, 0x00030002,
525 0x918c, 0xffffffff, 0x00010004,
526 0x9190, 0xffffffff, 0x00070006,
527 0x9194, 0xffffffff, 0x00050000,
528 0x9198, 0xffffffff, 0x00030002,
529 0x91a8, 0xffffffff, 0x00010004,
530 0x91ac, 0xffffffff, 0x00070006,
531 0x91e8, 0xffffffff, 0x00000000,
532 0x9294, 0xffffffff, 0x00000000,
533 0x929c, 0xffffffff, 0x00000001,
534 0x802c, 0xffffffff, 0xc0000000
537 static const u32 juniper_mgcg_init[] =
539 0x802c, 0xffffffff, 0xc0000000,
540 0x5448, 0xffffffff, 0x00000100,
541 0x55e4, 0xffffffff, 0x00000100,
542 0x160c, 0xffffffff, 0x00000100,
543 0x5644, 0xffffffff, 0x00000100,
544 0xc164, 0xffffffff, 0x00000100,
545 0x8a18, 0xffffffff, 0x00000100,
546 0x897c, 0xffffffff, 0x06000100,
547 0x8b28, 0xffffffff, 0x00000100,
548 0x9144, 0xffffffff, 0x00000100,
549 0x9a60, 0xffffffff, 0x00000100,
550 0x9868, 0xffffffff, 0x00000100,
551 0x8d58, 0xffffffff, 0x00000100,
552 0x9510, 0xffffffff, 0x00000100,
553 0x949c, 0xffffffff, 0x00000100,
554 0x9654, 0xffffffff, 0x00000100,
555 0x9030, 0xffffffff, 0x00000100,
556 0x9034, 0xffffffff, 0x00000100,
557 0x9038, 0xffffffff, 0x00000100,
558 0x903c, 0xffffffff, 0x00000100,
559 0x9040, 0xffffffff, 0x00000100,
560 0xa200, 0xffffffff, 0x00000100,
561 0xa204, 0xffffffff, 0x00000100,
562 0xa208, 0xffffffff, 0x00000100,
563 0xa20c, 0xffffffff, 0x00000100,
564 0x971c, 0xffffffff, 0x00000100,
565 0xd0c0, 0xffffffff, 0xff000100,
566 0x802c, 0xffffffff, 0x40000000,
567 0x915c, 0xffffffff, 0x00010000,
568 0x9160, 0xffffffff, 0x00030002,
569 0x9178, 0xffffffff, 0x00070000,
570 0x917c, 0xffffffff, 0x00030002,
571 0x9180, 0xffffffff, 0x00050004,
572 0x918c, 0xffffffff, 0x00010006,
573 0x9190, 0xffffffff, 0x00090008,
574 0x9194, 0xffffffff, 0x00070000,
575 0x9198, 0xffffffff, 0x00030002,
576 0x919c, 0xffffffff, 0x00050004,
577 0x91a8, 0xffffffff, 0x00010006,
578 0x91ac, 0xffffffff, 0x00090008,
579 0x91b0, 0xffffffff, 0x00070000,
580 0x91b4, 0xffffffff, 0x00030002,
581 0x91b8, 0xffffffff, 0x00050004,
582 0x91c4, 0xffffffff, 0x00010006,
583 0x91c8, 0xffffffff, 0x00090008,
584 0x91cc, 0xffffffff, 0x00070000,
585 0x91d0, 0xffffffff, 0x00030002,
586 0x91d4, 0xffffffff, 0x00050004,
587 0x91e0, 0xffffffff, 0x00010006,
588 0x91e4, 0xffffffff, 0x00090008,
589 0x91e8, 0xffffffff, 0x00000000,
590 0x91ec, 0xffffffff, 0x00070000,
591 0x91f0, 0xffffffff, 0x00030002,
592 0x91f4, 0xffffffff, 0x00050004,
593 0x9200, 0xffffffff, 0x00010006,
594 0x9204, 0xffffffff, 0x00090008,
595 0x9208, 0xffffffff, 0x00070000,
596 0x920c, 0xffffffff, 0x00030002,
597 0x9210, 0xffffffff, 0x00050004,
598 0x921c, 0xffffffff, 0x00010006,
599 0x9220, 0xffffffff, 0x00090008,
600 0x9224, 0xffffffff, 0x00070000,
601 0x9228, 0xffffffff, 0x00030002,
602 0x922c, 0xffffffff, 0x00050004,
603 0x9238, 0xffffffff, 0x00010006,
604 0x923c, 0xffffffff, 0x00090008,
605 0x9240, 0xffffffff, 0x00070000,
606 0x9244, 0xffffffff, 0x00030002,
607 0x9248, 0xffffffff, 0x00050004,
608 0x9254, 0xffffffff, 0x00010006,
609 0x9258, 0xffffffff, 0x00090008,
610 0x925c, 0xffffffff, 0x00070000,
611 0x9260, 0xffffffff, 0x00030002,
612 0x9264, 0xffffffff, 0x00050004,
613 0x9270, 0xffffffff, 0x00010006,
614 0x9274, 0xffffffff, 0x00090008,
615 0x9278, 0xffffffff, 0x00070000,
616 0x927c, 0xffffffff, 0x00030002,
617 0x9280, 0xffffffff, 0x00050004,
618 0x928c, 0xffffffff, 0x00010006,
619 0x9290, 0xffffffff, 0x00090008,
620 0x9294, 0xffffffff, 0x00000000,
621 0x929c, 0xffffffff, 0x00000001,
622 0x802c, 0xffffffff, 0xc0000000,
623 0x977c, 0xffffffff, 0x00000100,
624 0x3f80, 0xffffffff, 0x00000100,
625 0xa210, 0xffffffff, 0x00000100,
626 0xa214, 0xffffffff, 0x00000100,
627 0x4d8, 0xffffffff, 0x00000100,
628 0x9784, 0xffffffff, 0x00000100,
629 0x9698, 0xffffffff, 0x00000100,
630 0x4d4, 0xffffffff, 0x00000200,
631 0x30cc, 0xffffffff, 0x00000100,
632 0x802c, 0xffffffff, 0xc0000000
635 static const u32 supersumo_golden_registers[] =
637 0x5eb4, 0xffffffff, 0x00000002,
638 0x5c4, 0xffffffff, 0x00000001,
639 0x7030, 0xffffffff, 0x00000011,
640 0x7c30, 0xffffffff, 0x00000011,
641 0x6104, 0x01000300, 0x00000000,
642 0x5bc0, 0x00300000, 0x00000000,
643 0x8c04, 0xffffffff, 0x40600060,
644 0x8c08, 0xffffffff, 0x001c001c,
645 0x8c20, 0xffffffff, 0x00800080,
646 0x8c24, 0xffffffff, 0x00800080,
647 0x8c18, 0xffffffff, 0x20202078,
648 0x8c1c, 0xffffffff, 0x00001010,
649 0x918c, 0xffffffff, 0x00010006,
650 0x91a8, 0xffffffff, 0x00010006,
651 0x91c4, 0xffffffff, 0x00010006,
652 0x91e0, 0xffffffff, 0x00010006,
653 0x9200, 0xffffffff, 0x00010006,
654 0x9150, 0xffffffff, 0x6e944040,
655 0x917c, 0xffffffff, 0x00030002,
656 0x9180, 0xffffffff, 0x00050004,
657 0x9198, 0xffffffff, 0x00030002,
658 0x919c, 0xffffffff, 0x00050004,
659 0x91b4, 0xffffffff, 0x00030002,
660 0x91b8, 0xffffffff, 0x00050004,
661 0x91d0, 0xffffffff, 0x00030002,
662 0x91d4, 0xffffffff, 0x00050004,
663 0x91f0, 0xffffffff, 0x00030002,
664 0x91f4, 0xffffffff, 0x00050004,
665 0x915c, 0xffffffff, 0x00010000,
666 0x9160, 0xffffffff, 0x00030002,
667 0x3f90, 0xffff0000, 0xff000000,
668 0x9178, 0xffffffff, 0x00070000,
669 0x9194, 0xffffffff, 0x00070000,
670 0x91b0, 0xffffffff, 0x00070000,
671 0x91cc, 0xffffffff, 0x00070000,
672 0x91ec, 0xffffffff, 0x00070000,
673 0x9148, 0xffff0000, 0xff000000,
674 0x9190, 0xffffffff, 0x00090008,
675 0x91ac, 0xffffffff, 0x00090008,
676 0x91c8, 0xffffffff, 0x00090008,
677 0x91e4, 0xffffffff, 0x00090008,
678 0x9204, 0xffffffff, 0x00090008,
679 0x3f94, 0xffff0000, 0xff000000,
680 0x914c, 0xffff0000, 0xff000000,
681 0x929c, 0xffffffff, 0x00000001,
682 0x8a18, 0xffffffff, 0x00000100,
683 0x8b28, 0xffffffff, 0x00000100,
684 0x9144, 0xffffffff, 0x00000100,
685 0x5644, 0xffffffff, 0x00000100,
686 0x9b7c, 0xffffffff, 0x00000000,
687 0x8030, 0xffffffff, 0x0000100a,
688 0x8a14, 0xffffffff, 0x00000007,
689 0x8b24, 0xffffffff, 0x00ff0fff,
690 0x8b10, 0xffffffff, 0x00000000,
691 0x28a4c, 0x06000000, 0x06000000,
692 0x4d8, 0xffffffff, 0x00000100,
693 0x913c, 0xffff000f, 0x0100000a,
694 0x960c, 0xffffffff, 0x54763210,
695 0x88c4, 0xffffffff, 0x000000c2,
696 0x88d4, 0xffffffff, 0x00000010,
697 0x8974, 0xffffffff, 0x00000000,
698 0xc78, 0x00000080, 0x00000080,
699 0x5e78, 0xffffffff, 0x001000f0,
700 0xd02c, 0xffffffff, 0x08421000,
701 0xa008, 0xffffffff, 0x00010000,
702 0x8d00, 0xffffffff, 0x100e4848,
703 0x8d04, 0xffffffff, 0x00164745,
704 0x8c00, 0xffffffff, 0xe4000003,
705 0x8cf0, 0x1fffffff, 0x08e00620,
706 0x28350, 0xffffffff, 0x00000000,
707 0x9508, 0xffffffff, 0x00000002
710 static const u32 sumo_golden_registers[] =
712 0x900c, 0x00ffffff, 0x0017071f,
713 0x8c18, 0xffffffff, 0x10101060,
714 0x8c1c, 0xffffffff, 0x00001010,
715 0x8c30, 0x0000000f, 0x00000005,
716 0x9688, 0x0000000f, 0x00000007
719 static const u32 wrestler_golden_registers[] =
721 0x5eb4, 0xffffffff, 0x00000002,
722 0x5c4, 0xffffffff, 0x00000001,
723 0x7030, 0xffffffff, 0x00000011,
724 0x7c30, 0xffffffff, 0x00000011,
725 0x6104, 0x01000300, 0x00000000,
726 0x5bc0, 0x00300000, 0x00000000,
727 0x918c, 0xffffffff, 0x00010006,
728 0x91a8, 0xffffffff, 0x00010006,
729 0x9150, 0xffffffff, 0x6e944040,
730 0x917c, 0xffffffff, 0x00030002,
731 0x9198, 0xffffffff, 0x00030002,
732 0x915c, 0xffffffff, 0x00010000,
733 0x3f90, 0xffff0000, 0xff000000,
734 0x9178, 0xffffffff, 0x00070000,
735 0x9194, 0xffffffff, 0x00070000,
736 0x9148, 0xffff0000, 0xff000000,
737 0x9190, 0xffffffff, 0x00090008,
738 0x91ac, 0xffffffff, 0x00090008,
739 0x3f94, 0xffff0000, 0xff000000,
740 0x914c, 0xffff0000, 0xff000000,
741 0x929c, 0xffffffff, 0x00000001,
742 0x8a18, 0xffffffff, 0x00000100,
743 0x8b28, 0xffffffff, 0x00000100,
744 0x9144, 0xffffffff, 0x00000100,
745 0x9b7c, 0xffffffff, 0x00000000,
746 0x8030, 0xffffffff, 0x0000100a,
747 0x8a14, 0xffffffff, 0x00000001,
748 0x8b24, 0xffffffff, 0x00ff0fff,
749 0x8b10, 0xffffffff, 0x00000000,
750 0x28a4c, 0x06000000, 0x06000000,
751 0x4d8, 0xffffffff, 0x00000100,
752 0x913c, 0xffff000f, 0x0100000a,
753 0x960c, 0xffffffff, 0x54763210,
754 0x88c4, 0xffffffff, 0x000000c2,
755 0x88d4, 0xffffffff, 0x00000010,
756 0x8974, 0xffffffff, 0x00000000,
757 0xc78, 0x00000080, 0x00000080,
758 0x5e78, 0xffffffff, 0x001000f0,
759 0xd02c, 0xffffffff, 0x08421000,
760 0xa008, 0xffffffff, 0x00010000,
761 0x8d00, 0xffffffff, 0x100e4848,
762 0x8d04, 0xffffffff, 0x00164745,
763 0x8c00, 0xffffffff, 0xe4000003,
764 0x8cf0, 0x1fffffff, 0x08e00410,
765 0x28350, 0xffffffff, 0x00000000,
766 0x9508, 0xffffffff, 0x00000002,
767 0x900c, 0xffffffff, 0x0017071f,
768 0x8c18, 0xffffffff, 0x10101060,
769 0x8c1c, 0xffffffff, 0x00001010
772 static const u32 barts_golden_registers[] =
774 0x5eb4, 0xffffffff, 0x00000002,
775 0x5e78, 0x8f311ff1, 0x001000f0,
776 0x3f90, 0xffff0000, 0xff000000,
777 0x9148, 0xffff0000, 0xff000000,
778 0x3f94, 0xffff0000, 0xff000000,
779 0x914c, 0xffff0000, 0xff000000,
780 0xc78, 0x00000080, 0x00000080,
781 0xbd4, 0x70073777, 0x00010001,
782 0xd02c, 0xbfffff1f, 0x08421000,
783 0xd0b8, 0x03773777, 0x02011003,
784 0x5bc0, 0x00200000, 0x50100000,
785 0x98f8, 0x33773777, 0x02011003,
786 0x98fc, 0xffffffff, 0x76543210,
787 0x7030, 0x31000311, 0x00000011,
788 0x2f48, 0x00000007, 0x02011003,
789 0x6b28, 0x00000010, 0x00000012,
790 0x7728, 0x00000010, 0x00000012,
791 0x10328, 0x00000010, 0x00000012,
792 0x10f28, 0x00000010, 0x00000012,
793 0x11b28, 0x00000010, 0x00000012,
794 0x12728, 0x00000010, 0x00000012,
795 0x240c, 0x000007ff, 0x00000380,
796 0x8a14, 0xf000001f, 0x00000007,
797 0x8b24, 0x3fff3fff, 0x00ff0fff,
798 0x8b10, 0x0000ff0f, 0x00000000,
799 0x28a4c, 0x07ffffff, 0x06000000,
800 0x10c, 0x00000001, 0x00010003,
801 0xa02c, 0xffffffff, 0x0000009b,
802 0x913c, 0x0000000f, 0x0100000a,
803 0x8d00, 0xffff7f7f, 0x100e4848,
804 0x8d04, 0x00ffffff, 0x00164745,
805 0x8c00, 0xfffc0003, 0xe4000003,
806 0x8c04, 0xf8ff00ff, 0x40600060,
807 0x8c08, 0x00ff00ff, 0x001c001c,
808 0x8cf0, 0x1fff1fff, 0x08e00620,
809 0x8c20, 0x0fff0fff, 0x00800080,
810 0x8c24, 0x0fff0fff, 0x00800080,
811 0x8c18, 0xffffffff, 0x20202078,
812 0x8c1c, 0x0000ffff, 0x00001010,
813 0x28350, 0x00000f01, 0x00000000,
814 0x9508, 0x3700001f, 0x00000002,
815 0x960c, 0xffffffff, 0x54763210,
816 0x88c4, 0x001f3ae3, 0x000000c2,
817 0x88d4, 0x0000001f, 0x00000010,
818 0x8974, 0xffffffff, 0x00000000
821 static const u32 turks_golden_registers[] =
823 0x5eb4, 0xffffffff, 0x00000002,
824 0x5e78, 0x8f311ff1, 0x001000f0,
825 0x8c8, 0x00003000, 0x00001070,
826 0x8cc, 0x000fffff, 0x00040035,
827 0x3f90, 0xffff0000, 0xfff00000,
828 0x9148, 0xffff0000, 0xfff00000,
829 0x3f94, 0xffff0000, 0xfff00000,
830 0x914c, 0xffff0000, 0xfff00000,
831 0xc78, 0x00000080, 0x00000080,
832 0xbd4, 0x00073007, 0x00010002,
833 0xd02c, 0xbfffff1f, 0x08421000,
834 0xd0b8, 0x03773777, 0x02010002,
835 0x5bc0, 0x00200000, 0x50100000,
836 0x98f8, 0x33773777, 0x00010002,
837 0x98fc, 0xffffffff, 0x33221100,
838 0x7030, 0x31000311, 0x00000011,
839 0x2f48, 0x33773777, 0x00010002,
840 0x6b28, 0x00000010, 0x00000012,
841 0x7728, 0x00000010, 0x00000012,
842 0x10328, 0x00000010, 0x00000012,
843 0x10f28, 0x00000010, 0x00000012,
844 0x11b28, 0x00000010, 0x00000012,
845 0x12728, 0x00000010, 0x00000012,
846 0x240c, 0x000007ff, 0x00000380,
847 0x8a14, 0xf000001f, 0x00000007,
848 0x8b24, 0x3fff3fff, 0x00ff0fff,
849 0x8b10, 0x0000ff0f, 0x00000000,
850 0x28a4c, 0x07ffffff, 0x06000000,
851 0x10c, 0x00000001, 0x00010003,
852 0xa02c, 0xffffffff, 0x0000009b,
853 0x913c, 0x0000000f, 0x0100000a,
854 0x8d00, 0xffff7f7f, 0x100e4848,
855 0x8d04, 0x00ffffff, 0x00164745,
856 0x8c00, 0xfffc0003, 0xe4000003,
857 0x8c04, 0xf8ff00ff, 0x40600060,
858 0x8c08, 0x00ff00ff, 0x001c001c,
859 0x8cf0, 0x1fff1fff, 0x08e00410,
860 0x8c20, 0x0fff0fff, 0x00800080,
861 0x8c24, 0x0fff0fff, 0x00800080,
862 0x8c18, 0xffffffff, 0x20202078,
863 0x8c1c, 0x0000ffff, 0x00001010,
864 0x28350, 0x00000f01, 0x00000000,
865 0x9508, 0x3700001f, 0x00000002,
866 0x960c, 0xffffffff, 0x54763210,
867 0x88c4, 0x001f3ae3, 0x000000c2,
868 0x88d4, 0x0000001f, 0x00000010,
869 0x8974, 0xffffffff, 0x00000000
872 static const u32 caicos_golden_registers[] =
874 0x5eb4, 0xffffffff, 0x00000002,
875 0x5e78, 0x8f311ff1, 0x001000f0,
876 0x8c8, 0x00003420, 0x00001450,
877 0x8cc, 0x000fffff, 0x00040035,
878 0x3f90, 0xffff0000, 0xfffc0000,
879 0x9148, 0xffff0000, 0xfffc0000,
880 0x3f94, 0xffff0000, 0xfffc0000,
881 0x914c, 0xffff0000, 0xfffc0000,
882 0xc78, 0x00000080, 0x00000080,
883 0xbd4, 0x00073007, 0x00010001,
884 0xd02c, 0xbfffff1f, 0x08421000,
885 0xd0b8, 0x03773777, 0x02010001,
886 0x5bc0, 0x00200000, 0x50100000,
887 0x98f8, 0x33773777, 0x02010001,
888 0x98fc, 0xffffffff, 0x33221100,
889 0x7030, 0x31000311, 0x00000011,
890 0x2f48, 0x33773777, 0x02010001,
891 0x6b28, 0x00000010, 0x00000012,
892 0x7728, 0x00000010, 0x00000012,
893 0x10328, 0x00000010, 0x00000012,
894 0x10f28, 0x00000010, 0x00000012,
895 0x11b28, 0x00000010, 0x00000012,
896 0x12728, 0x00000010, 0x00000012,
897 0x240c, 0x000007ff, 0x00000380,
898 0x8a14, 0xf000001f, 0x00000001,
899 0x8b24, 0x3fff3fff, 0x00ff0fff,
900 0x8b10, 0x0000ff0f, 0x00000000,
901 0x28a4c, 0x07ffffff, 0x06000000,
902 0x10c, 0x00000001, 0x00010003,
903 0xa02c, 0xffffffff, 0x0000009b,
904 0x913c, 0x0000000f, 0x0100000a,
905 0x8d00, 0xffff7f7f, 0x100e4848,
906 0x8d04, 0x00ffffff, 0x00164745,
907 0x8c00, 0xfffc0003, 0xe4000003,
908 0x8c04, 0xf8ff00ff, 0x40600060,
909 0x8c08, 0x00ff00ff, 0x001c001c,
910 0x8cf0, 0x1fff1fff, 0x08e00410,
911 0x8c20, 0x0fff0fff, 0x00800080,
912 0x8c24, 0x0fff0fff, 0x00800080,
913 0x8c18, 0xffffffff, 0x20202078,
914 0x8c1c, 0x0000ffff, 0x00001010,
915 0x28350, 0x00000f01, 0x00000000,
916 0x9508, 0x3700001f, 0x00000002,
917 0x960c, 0xffffffff, 0x54763210,
918 0x88c4, 0x001f3ae3, 0x000000c2,
919 0x88d4, 0x0000001f, 0x00000010,
920 0x8974, 0xffffffff, 0x00000000
923 static void evergreen_init_golden_registers(struct radeon_device *rdev)
925 switch (rdev->family) {
928 radeon_program_register_sequence(rdev,
929 evergreen_golden_registers,
930 (const u32)ARRAY_SIZE(evergreen_golden_registers));
931 radeon_program_register_sequence(rdev,
932 evergreen_golden_registers2,
933 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
934 radeon_program_register_sequence(rdev,
936 (const u32)ARRAY_SIZE(cypress_mgcg_init));
939 radeon_program_register_sequence(rdev,
940 evergreen_golden_registers,
941 (const u32)ARRAY_SIZE(evergreen_golden_registers));
942 radeon_program_register_sequence(rdev,
943 evergreen_golden_registers2,
944 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
945 radeon_program_register_sequence(rdev,
947 (const u32)ARRAY_SIZE(juniper_mgcg_init));
950 radeon_program_register_sequence(rdev,
951 evergreen_golden_registers,
952 (const u32)ARRAY_SIZE(evergreen_golden_registers));
953 radeon_program_register_sequence(rdev,
954 evergreen_golden_registers2,
955 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
956 radeon_program_register_sequence(rdev,
958 (const u32)ARRAY_SIZE(redwood_mgcg_init));
961 radeon_program_register_sequence(rdev,
962 cedar_golden_registers,
963 (const u32)ARRAY_SIZE(cedar_golden_registers));
964 radeon_program_register_sequence(rdev,
965 evergreen_golden_registers2,
966 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
967 radeon_program_register_sequence(rdev,
969 (const u32)ARRAY_SIZE(cedar_mgcg_init));
972 radeon_program_register_sequence(rdev,
973 wrestler_golden_registers,
974 (const u32)ARRAY_SIZE(wrestler_golden_registers));
977 radeon_program_register_sequence(rdev,
978 supersumo_golden_registers,
979 (const u32)ARRAY_SIZE(supersumo_golden_registers));
982 radeon_program_register_sequence(rdev,
983 supersumo_golden_registers,
984 (const u32)ARRAY_SIZE(supersumo_golden_registers));
985 radeon_program_register_sequence(rdev,
986 sumo_golden_registers,
987 (const u32)ARRAY_SIZE(sumo_golden_registers));
990 radeon_program_register_sequence(rdev,
991 barts_golden_registers,
992 (const u32)ARRAY_SIZE(barts_golden_registers));
995 radeon_program_register_sequence(rdev,
996 turks_golden_registers,
997 (const u32)ARRAY_SIZE(turks_golden_registers));
1000 radeon_program_register_sequence(rdev,
1001 caicos_golden_registers,
1002 (const u32)ARRAY_SIZE(caicos_golden_registers));
1009 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1010 unsigned *bankh, unsigned *mtaspect,
1011 unsigned *tile_split)
1013 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1014 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1015 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1016 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1019 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1020 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1021 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1022 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1026 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1027 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1028 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1029 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1031 switch (*mtaspect) {
1033 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1034 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1035 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1036 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1040 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1041 u32 cntl_reg, u32 status_reg)
1044 struct atom_clock_dividers dividers;
1046 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1047 clock, false, ÷rs);
1051 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1053 for (i = 0; i < 100; i++) {
1054 if (RREG32(status_reg) & DCLK_STATUS)
1064 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1067 u32 cg_scratch = RREG32(CG_SCRATCH1);
1069 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1072 cg_scratch &= 0xffff0000;
1073 cg_scratch |= vclk / 100; /* Mhz */
1075 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1078 cg_scratch &= 0x0000ffff;
1079 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1082 WREG32(CG_SCRATCH1, cg_scratch);
1087 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1089 /* start off with something large */
1090 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1093 /* bypass vclk and dclk with bclk */
1094 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1095 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1096 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1098 /* put PLL in bypass mode */
1099 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1101 if (!vclk || !dclk) {
1102 /* keep the Bypass mode, put PLL to sleep */
1103 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1107 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1108 16384, 0x03FFFFFF, 0, 128, 5,
1109 &fb_div, &vclk_div, &dclk_div);
1113 /* set VCO_MODE to 1 */
1114 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1116 /* toggle UPLL_SLEEP to 1 then back to 0 */
1117 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1118 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1120 /* deassert UPLL_RESET */
1121 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1125 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1129 /* assert UPLL_RESET again */
1130 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1132 /* disable spread spectrum. */
1133 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1135 /* set feedback divider */
1136 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1138 /* set ref divider to 0 */
1139 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1141 if (fb_div < 307200)
1142 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1144 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1146 /* set PDIV_A and PDIV_B */
1147 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1148 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1149 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1151 /* give the PLL some time to settle */
1154 /* deassert PLL_RESET */
1155 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1159 /* switch from bypass mode to normal mode */
1160 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1162 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1166 /* switch VCLK and DCLK selection */
1167 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1168 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1169 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1176 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1181 readrq = pcie_get_readrq(rdev->pdev);
1182 v = ffs(readrq) - 8;
1183 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1184 * to avoid hangs or perfomance issues
1186 if ((v == 0) || (v == 6) || (v == 7))
1187 pcie_set_readrq(rdev->pdev, 512);
1190 void dce4_program_fmt(struct drm_encoder *encoder)
1192 struct drm_device *dev = encoder->dev;
1193 struct radeon_device *rdev = dev->dev_private;
1194 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1195 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1196 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1199 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1202 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1203 bpc = radeon_get_monitor_bpc(connector);
1204 dither = radeon_connector->dither;
1207 /* LVDS/eDP FMT is set up by atom */
1208 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1211 /* not needed for analog */
1212 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1213 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1221 if (dither == RADEON_FMT_DITHER_ENABLE)
1222 /* XXX sort out optimal dither settings */
1223 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1224 FMT_SPATIAL_DITHER_EN);
1226 tmp |= FMT_TRUNCATE_EN;
1229 if (dither == RADEON_FMT_DITHER_ENABLE)
1230 /* XXX sort out optimal dither settings */
1231 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1232 FMT_RGB_RANDOM_ENABLE |
1233 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1235 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1243 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1246 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1248 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1254 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1258 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1259 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1268 * dce4_wait_for_vblank - vblank wait asic callback.
1270 * @rdev: radeon_device pointer
1271 * @crtc: crtc to wait for vblank on
1273 * Wait for vblank on the requested crtc (evergreen+).
1275 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1279 if (crtc >= rdev->num_crtc)
1282 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1285 /* depending on when we hit vblank, we may be close to active; if so,
1286 * wait for another frame.
1288 while (dce4_is_in_vblank(rdev, crtc)) {
1289 if (i++ % 100 == 0) {
1290 if (!dce4_is_counter_moving(rdev, crtc))
1295 while (!dce4_is_in_vblank(rdev, crtc)) {
1296 if (i++ % 100 == 0) {
1297 if (!dce4_is_counter_moving(rdev, crtc))
1304 * evergreen_page_flip - pageflip callback.
1306 * @rdev: radeon_device pointer
1307 * @crtc_id: crtc to cleanup pageflip on
1308 * @crtc_base: new address of the crtc (GPU MC address)
1310 * Does the actual pageflip (evergreen+).
1311 * During vblank we take the crtc lock and wait for the update_pending
1312 * bit to go high, when it does, we release the lock, and allow the
1313 * double buffered update to take place.
1314 * Returns the current update pending status.
1316 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1318 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1319 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1322 /* Lock the graphics update lock */
1323 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1324 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1326 /* update the scanout addresses */
1327 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1328 upper_32_bits(crtc_base));
1329 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1332 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1333 upper_32_bits(crtc_base));
1334 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1337 /* Wait for update_pending to go high. */
1338 for (i = 0; i < rdev->usec_timeout; i++) {
1339 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1343 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1345 /* Unlock the lock, so double-buffering can take place inside vblank */
1346 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1347 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1351 * evergreen_page_flip_pending - check if page flip is still pending
1353 * @rdev: radeon_device pointer
1354 * @crtc_id: crtc to check
1356 * Returns the current update pending status.
1358 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1360 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1362 /* Return current update_pending status: */
1363 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1364 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1367 /* get temperature in millidegrees */
1368 int evergreen_get_temp(struct radeon_device *rdev)
1371 int actual_temp = 0;
1373 if (rdev->family == CHIP_JUNIPER) {
1374 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1376 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1379 if (toffset & 0x100)
1380 actual_temp = temp / 2 - (0x200 - toffset);
1382 actual_temp = temp / 2 + toffset;
1384 actual_temp = actual_temp * 1000;
1387 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1392 else if (temp & 0x200)
1394 else if (temp & 0x100) {
1395 actual_temp = temp & 0x1ff;
1396 actual_temp |= ~0x1ff;
1398 actual_temp = temp & 0xff;
1400 actual_temp = (actual_temp * 1000) / 2;
1406 int sumo_get_temp(struct radeon_device *rdev)
1408 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1409 int actual_temp = temp - 49;
1411 return actual_temp * 1000;
1415 * sumo_pm_init_profile - Initialize power profiles callback.
1417 * @rdev: radeon_device pointer
1419 * Initialize the power states used in profile mode
1420 * (sumo, trinity, SI).
1421 * Used for profile mode only.
1423 void sumo_pm_init_profile(struct radeon_device *rdev)
1428 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1429 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1430 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1431 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1434 if (rdev->flags & RADEON_IS_MOBILITY)
1435 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1437 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1439 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1440 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1441 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1442 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1444 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1445 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1446 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1447 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1449 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1450 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1451 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1452 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1454 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1455 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1456 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1457 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1460 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1461 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1462 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1463 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1464 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1465 rdev->pm.power_state[idx].num_clock_modes - 1;
1467 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1469 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1470 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1471 rdev->pm.power_state[idx].num_clock_modes - 1;
1475 * btc_pm_init_profile - Initialize power profiles callback.
1477 * @rdev: radeon_device pointer
1479 * Initialize the power states used in profile mode
1481 * Used for profile mode only.
1483 void btc_pm_init_profile(struct radeon_device *rdev)
1488 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1489 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1490 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1491 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1492 /* starting with BTC, there is one state that is used for both
1493 * MH and SH. Difference is that we always use the high clock index for
1496 if (rdev->flags & RADEON_IS_MOBILITY)
1497 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1499 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1501 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1502 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1503 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1504 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1506 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1507 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1508 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1509 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1511 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1512 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1513 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1514 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1516 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1517 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1518 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1519 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1521 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1522 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1523 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1524 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1526 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1529 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1533 * evergreen_pm_misc - set additional pm hw parameters callback.
1535 * @rdev: radeon_device pointer
1537 * Set non-clock parameters associated with a power state
1538 * (voltage, etc.) (evergreen+).
1540 void evergreen_pm_misc(struct radeon_device *rdev)
1542 int req_ps_idx = rdev->pm.requested_power_state_index;
1543 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1544 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1545 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1547 if (voltage->type == VOLTAGE_SW) {
1548 /* 0xff0x are flags rather then an actual voltage */
1549 if ((voltage->voltage & 0xff00) == 0xff00)
1551 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1552 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1553 rdev->pm.current_vddc = voltage->voltage;
1554 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1557 /* starting with BTC, there is one state that is used for both
1558 * MH and SH. Difference is that we always use the high clock index for
1561 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1562 (rdev->family >= CHIP_BARTS) &&
1563 rdev->pm.active_crtc_count &&
1564 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1565 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1566 voltage = &rdev->pm.power_state[req_ps_idx].
1567 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1569 /* 0xff0x are flags rather then an actual voltage */
1570 if ((voltage->vddci & 0xff00) == 0xff00)
1572 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1573 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1574 rdev->pm.current_vddci = voltage->vddci;
1575 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1581 * evergreen_pm_prepare - pre-power state change callback.
1583 * @rdev: radeon_device pointer
1585 * Prepare for a power state change (evergreen+).
1587 void evergreen_pm_prepare(struct radeon_device *rdev)
1589 struct drm_device *ddev = rdev->ddev;
1590 struct drm_crtc *crtc;
1591 struct radeon_crtc *radeon_crtc;
1594 /* disable any active CRTCs */
1595 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1596 radeon_crtc = to_radeon_crtc(crtc);
1597 if (radeon_crtc->enabled) {
1598 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1599 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1600 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1606 * evergreen_pm_finish - post-power state change callback.
1608 * @rdev: radeon_device pointer
1610 * Clean up after a power state change (evergreen+).
1612 void evergreen_pm_finish(struct radeon_device *rdev)
1614 struct drm_device *ddev = rdev->ddev;
1615 struct drm_crtc *crtc;
1616 struct radeon_crtc *radeon_crtc;
1619 /* enable any active CRTCs */
1620 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1621 radeon_crtc = to_radeon_crtc(crtc);
1622 if (radeon_crtc->enabled) {
1623 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1624 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1625 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1631 * evergreen_hpd_sense - hpd sense callback.
1633 * @rdev: radeon_device pointer
1634 * @hpd: hpd (hotplug detect) pin
1636 * Checks if a digital monitor is connected (evergreen+).
1637 * Returns true if connected, false if not connected.
1639 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1641 bool connected = false;
1645 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1649 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1653 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1657 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1661 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1665 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1676 * evergreen_hpd_set_polarity - hpd set polarity callback.
1678 * @rdev: radeon_device pointer
1679 * @hpd: hpd (hotplug detect) pin
1681 * Set the polarity of the hpd pin (evergreen+).
1683 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1684 enum radeon_hpd_id hpd)
1687 bool connected = evergreen_hpd_sense(rdev, hpd);
1691 tmp = RREG32(DC_HPD1_INT_CONTROL);
1693 tmp &= ~DC_HPDx_INT_POLARITY;
1695 tmp |= DC_HPDx_INT_POLARITY;
1696 WREG32(DC_HPD1_INT_CONTROL, tmp);
1699 tmp = RREG32(DC_HPD2_INT_CONTROL);
1701 tmp &= ~DC_HPDx_INT_POLARITY;
1703 tmp |= DC_HPDx_INT_POLARITY;
1704 WREG32(DC_HPD2_INT_CONTROL, tmp);
1707 tmp = RREG32(DC_HPD3_INT_CONTROL);
1709 tmp &= ~DC_HPDx_INT_POLARITY;
1711 tmp |= DC_HPDx_INT_POLARITY;
1712 WREG32(DC_HPD3_INT_CONTROL, tmp);
1715 tmp = RREG32(DC_HPD4_INT_CONTROL);
1717 tmp &= ~DC_HPDx_INT_POLARITY;
1719 tmp |= DC_HPDx_INT_POLARITY;
1720 WREG32(DC_HPD4_INT_CONTROL, tmp);
1723 tmp = RREG32(DC_HPD5_INT_CONTROL);
1725 tmp &= ~DC_HPDx_INT_POLARITY;
1727 tmp |= DC_HPDx_INT_POLARITY;
1728 WREG32(DC_HPD5_INT_CONTROL, tmp);
1731 tmp = RREG32(DC_HPD6_INT_CONTROL);
1733 tmp &= ~DC_HPDx_INT_POLARITY;
1735 tmp |= DC_HPDx_INT_POLARITY;
1736 WREG32(DC_HPD6_INT_CONTROL, tmp);
1744 * evergreen_hpd_init - hpd setup callback.
1746 * @rdev: radeon_device pointer
1748 * Setup the hpd pins used by the card (evergreen+).
1749 * Enable the pin, set the polarity, and enable the hpd interrupts.
1751 void evergreen_hpd_init(struct radeon_device *rdev)
1753 struct drm_device *dev = rdev->ddev;
1754 struct drm_connector *connector;
1755 unsigned enabled = 0;
1756 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1757 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1759 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1760 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1762 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1763 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1764 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1765 * aux dp channel on imac and help (but not completely fix)
1766 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1767 * also avoid interrupt storms during dpms.
1771 switch (radeon_connector->hpd.hpd) {
1773 WREG32(DC_HPD1_CONTROL, tmp);
1776 WREG32(DC_HPD2_CONTROL, tmp);
1779 WREG32(DC_HPD3_CONTROL, tmp);
1782 WREG32(DC_HPD4_CONTROL, tmp);
1785 WREG32(DC_HPD5_CONTROL, tmp);
1788 WREG32(DC_HPD6_CONTROL, tmp);
1793 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1794 enabled |= 1 << radeon_connector->hpd.hpd;
1796 radeon_irq_kms_enable_hpd(rdev, enabled);
1800 * evergreen_hpd_fini - hpd tear down callback.
1802 * @rdev: radeon_device pointer
1804 * Tear down the hpd pins used by the card (evergreen+).
1805 * Disable the hpd interrupts.
1807 void evergreen_hpd_fini(struct radeon_device *rdev)
1809 struct drm_device *dev = rdev->ddev;
1810 struct drm_connector *connector;
1811 unsigned disabled = 0;
1813 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1814 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1815 switch (radeon_connector->hpd.hpd) {
1817 WREG32(DC_HPD1_CONTROL, 0);
1820 WREG32(DC_HPD2_CONTROL, 0);
1823 WREG32(DC_HPD3_CONTROL, 0);
1826 WREG32(DC_HPD4_CONTROL, 0);
1829 WREG32(DC_HPD5_CONTROL, 0);
1832 WREG32(DC_HPD6_CONTROL, 0);
1837 disabled |= 1 << radeon_connector->hpd.hpd;
1839 radeon_irq_kms_disable_hpd(rdev, disabled);
1842 /* watermark setup */
1844 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1845 struct radeon_crtc *radeon_crtc,
1846 struct drm_display_mode *mode,
1847 struct drm_display_mode *other_mode)
1849 u32 tmp, buffer_alloc, i;
1850 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1853 * There are 3 line buffers, each one shared by 2 display controllers.
1854 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1855 * the display controllers. The paritioning is done via one of four
1856 * preset allocations specified in bits 2:0:
1857 * first display controller
1858 * 0 - first half of lb (3840 * 2)
1859 * 1 - first 3/4 of lb (5760 * 2)
1860 * 2 - whole lb (7680 * 2), other crtc must be disabled
1861 * 3 - first 1/4 of lb (1920 * 2)
1862 * second display controller
1863 * 4 - second half of lb (3840 * 2)
1864 * 5 - second 3/4 of lb (5760 * 2)
1865 * 6 - whole lb (7680 * 2), other crtc must be disabled
1866 * 7 - last 1/4 of lb (1920 * 2)
1868 /* this can get tricky if we have two large displays on a paired group
1869 * of crtcs. Ideally for multiple large displays we'd assign them to
1870 * non-linked crtcs for maximum line buffer allocation.
1872 if (radeon_crtc->base.enabled && mode) {
1877 tmp = 2; /* whole */
1885 /* second controller of the pair uses second half of the lb */
1886 if (radeon_crtc->crtc_id % 2)
1888 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1890 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1891 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1892 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1893 for (i = 0; i < rdev->usec_timeout; i++) {
1894 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1895 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1901 if (radeon_crtc->base.enabled && mode) {
1906 if (ASIC_IS_DCE5(rdev))
1912 if (ASIC_IS_DCE5(rdev))
1918 if (ASIC_IS_DCE5(rdev))
1924 if (ASIC_IS_DCE5(rdev))
1931 /* controller not enabled, so no lb used */
1935 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1937 u32 tmp = RREG32(MC_SHARED_CHMAP);
1939 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1952 struct evergreen_wm_params {
1953 u32 dram_channels; /* number of dram channels */
1954 u32 yclk; /* bandwidth per dram data pin in kHz */
1955 u32 sclk; /* engine clock in kHz */
1956 u32 disp_clk; /* display clock in kHz */
1957 u32 src_width; /* viewport width */
1958 u32 active_time; /* active display time in ns */
1959 u32 blank_time; /* blank time in ns */
1960 bool interlaced; /* mode is interlaced */
1961 fixed20_12 vsc; /* vertical scale ratio */
1962 u32 num_heads; /* number of active crtcs */
1963 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1964 u32 lb_size; /* line buffer allocated to pipe */
1965 u32 vtaps; /* vertical scaler taps */
1968 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1970 /* Calculate DRAM Bandwidth and the part allocated to display. */
1971 fixed20_12 dram_efficiency; /* 0.7 */
1972 fixed20_12 yclk, dram_channels, bandwidth;
1975 a.full = dfixed_const(1000);
1976 yclk.full = dfixed_const(wm->yclk);
1977 yclk.full = dfixed_div(yclk, a);
1978 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1979 a.full = dfixed_const(10);
1980 dram_efficiency.full = dfixed_const(7);
1981 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1982 bandwidth.full = dfixed_mul(dram_channels, yclk);
1983 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1985 return dfixed_trunc(bandwidth);
1988 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1990 /* Calculate DRAM Bandwidth and the part allocated to display. */
1991 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1992 fixed20_12 yclk, dram_channels, bandwidth;
1995 a.full = dfixed_const(1000);
1996 yclk.full = dfixed_const(wm->yclk);
1997 yclk.full = dfixed_div(yclk, a);
1998 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1999 a.full = dfixed_const(10);
2000 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2001 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2002 bandwidth.full = dfixed_mul(dram_channels, yclk);
2003 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2005 return dfixed_trunc(bandwidth);
2008 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2010 /* Calculate the display Data return Bandwidth */
2011 fixed20_12 return_efficiency; /* 0.8 */
2012 fixed20_12 sclk, bandwidth;
2015 a.full = dfixed_const(1000);
2016 sclk.full = dfixed_const(wm->sclk);
2017 sclk.full = dfixed_div(sclk, a);
2018 a.full = dfixed_const(10);
2019 return_efficiency.full = dfixed_const(8);
2020 return_efficiency.full = dfixed_div(return_efficiency, a);
2021 a.full = dfixed_const(32);
2022 bandwidth.full = dfixed_mul(a, sclk);
2023 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2025 return dfixed_trunc(bandwidth);
2028 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2030 /* Calculate the DMIF Request Bandwidth */
2031 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2032 fixed20_12 disp_clk, bandwidth;
2035 a.full = dfixed_const(1000);
2036 disp_clk.full = dfixed_const(wm->disp_clk);
2037 disp_clk.full = dfixed_div(disp_clk, a);
2038 a.full = dfixed_const(10);
2039 disp_clk_request_efficiency.full = dfixed_const(8);
2040 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2041 a.full = dfixed_const(32);
2042 bandwidth.full = dfixed_mul(a, disp_clk);
2043 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2045 return dfixed_trunc(bandwidth);
2048 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2050 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2051 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2052 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2053 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2055 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2058 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2060 /* Calculate the display mode Average Bandwidth
2061 * DisplayMode should contain the source and destination dimensions,
2065 fixed20_12 line_time;
2066 fixed20_12 src_width;
2067 fixed20_12 bandwidth;
2070 a.full = dfixed_const(1000);
2071 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2072 line_time.full = dfixed_div(line_time, a);
2073 bpp.full = dfixed_const(wm->bytes_per_pixel);
2074 src_width.full = dfixed_const(wm->src_width);
2075 bandwidth.full = dfixed_mul(src_width, bpp);
2076 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2077 bandwidth.full = dfixed_div(bandwidth, line_time);
2079 return dfixed_trunc(bandwidth);
2082 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2084 /* First calcualte the latency in ns */
2085 u32 mc_latency = 2000; /* 2000 ns. */
2086 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2087 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2088 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2089 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2090 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2091 (wm->num_heads * cursor_line_pair_return_time);
2092 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2093 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2096 if (wm->num_heads == 0)
2099 a.full = dfixed_const(2);
2100 b.full = dfixed_const(1);
2101 if ((wm->vsc.full > a.full) ||
2102 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2104 ((wm->vsc.full >= a.full) && wm->interlaced))
2105 max_src_lines_per_dst_line = 4;
2107 max_src_lines_per_dst_line = 2;
2109 a.full = dfixed_const(available_bandwidth);
2110 b.full = dfixed_const(wm->num_heads);
2111 a.full = dfixed_div(a, b);
2113 b.full = dfixed_const(1000);
2114 c.full = dfixed_const(wm->disp_clk);
2115 b.full = dfixed_div(c, b);
2116 c.full = dfixed_const(wm->bytes_per_pixel);
2117 b.full = dfixed_mul(b, c);
2119 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2121 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2122 b.full = dfixed_const(1000);
2123 c.full = dfixed_const(lb_fill_bw);
2124 b.full = dfixed_div(c, b);
2125 a.full = dfixed_div(a, b);
2126 line_fill_time = dfixed_trunc(a);
2128 if (line_fill_time < wm->active_time)
2131 return latency + (line_fill_time - wm->active_time);
2135 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2137 if (evergreen_average_bandwidth(wm) <=
2138 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2144 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2146 if (evergreen_average_bandwidth(wm) <=
2147 (evergreen_available_bandwidth(wm) / wm->num_heads))
2153 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2155 u32 lb_partitions = wm->lb_size / wm->src_width;
2156 u32 line_time = wm->active_time + wm->blank_time;
2157 u32 latency_tolerant_lines;
2161 a.full = dfixed_const(1);
2162 if (wm->vsc.full > a.full)
2163 latency_tolerant_lines = 1;
2165 if (lb_partitions <= (wm->vtaps + 1))
2166 latency_tolerant_lines = 1;
2168 latency_tolerant_lines = 2;
2171 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2173 if (evergreen_latency_watermark(wm) <= latency_hiding)
2179 static void evergreen_program_watermarks(struct radeon_device *rdev,
2180 struct radeon_crtc *radeon_crtc,
2181 u32 lb_size, u32 num_heads)
2183 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2184 struct evergreen_wm_params wm_low, wm_high;
2188 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2189 u32 priority_a_mark = 0, priority_b_mark = 0;
2190 u32 priority_a_cnt = PRIORITY_OFF;
2191 u32 priority_b_cnt = PRIORITY_OFF;
2192 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2193 u32 tmp, arb_control3;
2196 if (radeon_crtc->base.enabled && num_heads && mode) {
2197 pixel_period = 1000000 / (u32)mode->clock;
2198 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2201 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2203 /* watermark for high clocks */
2204 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2206 radeon_dpm_get_mclk(rdev, false) * 10;
2208 radeon_dpm_get_sclk(rdev, false) * 10;
2210 wm_high.yclk = rdev->pm.current_mclk * 10;
2211 wm_high.sclk = rdev->pm.current_sclk * 10;
2214 wm_high.disp_clk = mode->clock;
2215 wm_high.src_width = mode->crtc_hdisplay;
2216 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2217 wm_high.blank_time = line_time - wm_high.active_time;
2218 wm_high.interlaced = false;
2219 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2220 wm_high.interlaced = true;
2221 wm_high.vsc = radeon_crtc->vsc;
2223 if (radeon_crtc->rmx_type != RMX_OFF)
2225 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2226 wm_high.lb_size = lb_size;
2227 wm_high.dram_channels = dram_channels;
2228 wm_high.num_heads = num_heads;
2230 /* watermark for low clocks */
2231 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2233 radeon_dpm_get_mclk(rdev, true) * 10;
2235 radeon_dpm_get_sclk(rdev, true) * 10;
2237 wm_low.yclk = rdev->pm.current_mclk * 10;
2238 wm_low.sclk = rdev->pm.current_sclk * 10;
2241 wm_low.disp_clk = mode->clock;
2242 wm_low.src_width = mode->crtc_hdisplay;
2243 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2244 wm_low.blank_time = line_time - wm_low.active_time;
2245 wm_low.interlaced = false;
2246 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2247 wm_low.interlaced = true;
2248 wm_low.vsc = radeon_crtc->vsc;
2250 if (radeon_crtc->rmx_type != RMX_OFF)
2252 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2253 wm_low.lb_size = lb_size;
2254 wm_low.dram_channels = dram_channels;
2255 wm_low.num_heads = num_heads;
2257 /* set for high clocks */
2258 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2259 /* set for low clocks */
2260 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2262 /* possibly force display priority to high */
2263 /* should really do this at mode validation time... */
2264 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2265 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2266 !evergreen_check_latency_hiding(&wm_high) ||
2267 (rdev->disp_priority == 2)) {
2268 DRM_DEBUG_KMS("force priority a to high\n");
2269 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2271 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2272 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2273 !evergreen_check_latency_hiding(&wm_low) ||
2274 (rdev->disp_priority == 2)) {
2275 DRM_DEBUG_KMS("force priority b to high\n");
2276 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2279 a.full = dfixed_const(1000);
2280 b.full = dfixed_const(mode->clock);
2281 b.full = dfixed_div(b, a);
2282 c.full = dfixed_const(latency_watermark_a);
2283 c.full = dfixed_mul(c, b);
2284 c.full = dfixed_mul(c, radeon_crtc->hsc);
2285 c.full = dfixed_div(c, a);
2286 a.full = dfixed_const(16);
2287 c.full = dfixed_div(c, a);
2288 priority_a_mark = dfixed_trunc(c);
2289 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2291 a.full = dfixed_const(1000);
2292 b.full = dfixed_const(mode->clock);
2293 b.full = dfixed_div(b, a);
2294 c.full = dfixed_const(latency_watermark_b);
2295 c.full = dfixed_mul(c, b);
2296 c.full = dfixed_mul(c, radeon_crtc->hsc);
2297 c.full = dfixed_div(c, a);
2298 a.full = dfixed_const(16);
2299 c.full = dfixed_div(c, a);
2300 priority_b_mark = dfixed_trunc(c);
2301 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2305 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2307 tmp &= ~LATENCY_WATERMARK_MASK(3);
2308 tmp |= LATENCY_WATERMARK_MASK(1);
2309 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2310 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2311 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2312 LATENCY_HIGH_WATERMARK(line_time)));
2314 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2315 tmp &= ~LATENCY_WATERMARK_MASK(3);
2316 tmp |= LATENCY_WATERMARK_MASK(2);
2317 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2318 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2319 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2320 LATENCY_HIGH_WATERMARK(line_time)));
2321 /* restore original selection */
2322 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2324 /* write the priority marks */
2325 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2326 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2328 /* save values for DPM */
2329 radeon_crtc->line_time = line_time;
2330 radeon_crtc->wm_high = latency_watermark_a;
2331 radeon_crtc->wm_low = latency_watermark_b;
2335 * evergreen_bandwidth_update - update display watermarks callback.
2337 * @rdev: radeon_device pointer
2339 * Update the display watermarks based on the requested mode(s)
2342 void evergreen_bandwidth_update(struct radeon_device *rdev)
2344 struct drm_display_mode *mode0 = NULL;
2345 struct drm_display_mode *mode1 = NULL;
2346 u32 num_heads = 0, lb_size;
2349 radeon_update_display_priority(rdev);
2351 for (i = 0; i < rdev->num_crtc; i++) {
2352 if (rdev->mode_info.crtcs[i]->base.enabled)
2355 for (i = 0; i < rdev->num_crtc; i += 2) {
2356 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2357 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2358 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2359 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2360 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2361 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2366 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2368 * @rdev: radeon_device pointer
2370 * Wait for the MC (memory controller) to be idle.
2372 * Returns 0 if the MC is idle, -1 if not.
2374 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2379 for (i = 0; i < rdev->usec_timeout; i++) {
2380 /* read MC_STATUS */
2381 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2392 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2397 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2399 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2400 for (i = 0; i < rdev->usec_timeout; i++) {
2401 /* read MC_STATUS */
2402 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2403 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2405 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2415 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2420 if (rdev->gart.robj == NULL) {
2421 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2424 r = radeon_gart_table_vram_pin(rdev);
2427 radeon_gart_restore(rdev);
2428 /* Setup L2 cache */
2429 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2430 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2431 EFFECTIVE_L2_QUEUE_SIZE(7));
2432 WREG32(VM_L2_CNTL2, 0);
2433 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2434 /* Setup TLB control */
2435 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2436 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2437 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2438 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2439 if (rdev->flags & RADEON_IS_IGP) {
2440 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2441 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2442 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2444 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2445 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2446 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2447 if ((rdev->family == CHIP_JUNIPER) ||
2448 (rdev->family == CHIP_CYPRESS) ||
2449 (rdev->family == CHIP_HEMLOCK) ||
2450 (rdev->family == CHIP_BARTS))
2451 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2453 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2454 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2455 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2456 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2457 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2458 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2459 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2460 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2461 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2462 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2463 (u32)(rdev->dummy_page.addr >> 12));
2464 WREG32(VM_CONTEXT1_CNTL, 0);
2466 evergreen_pcie_gart_tlb_flush(rdev);
2467 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2468 (unsigned)(rdev->mc.gtt_size >> 20),
2469 (unsigned long long)rdev->gart.table_addr);
2470 rdev->gart.ready = true;
2474 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2478 /* Disable all tables */
2479 WREG32(VM_CONTEXT0_CNTL, 0);
2480 WREG32(VM_CONTEXT1_CNTL, 0);
2482 /* Setup L2 cache */
2483 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2484 EFFECTIVE_L2_QUEUE_SIZE(7));
2485 WREG32(VM_L2_CNTL2, 0);
2486 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2487 /* Setup TLB control */
2488 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2489 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2490 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2491 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2492 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2493 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2494 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2495 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2496 radeon_gart_table_vram_unpin(rdev);
2499 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2501 evergreen_pcie_gart_disable(rdev);
2502 radeon_gart_table_vram_free(rdev);
2503 radeon_gart_fini(rdev);
2507 static void evergreen_agp_enable(struct radeon_device *rdev)
2511 /* Setup L2 cache */
2512 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2513 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2514 EFFECTIVE_L2_QUEUE_SIZE(7));
2515 WREG32(VM_L2_CNTL2, 0);
2516 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2517 /* Setup TLB control */
2518 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2519 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2520 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2521 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2522 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2523 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2524 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2525 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2526 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2527 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2528 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2529 WREG32(VM_CONTEXT0_CNTL, 0);
2530 WREG32(VM_CONTEXT1_CNTL, 0);
2533 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2535 u32 crtc_enabled, tmp, frame_count, blackout;
2538 if (!ASIC_IS_NODCE(rdev)) {
2539 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2540 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2542 /* disable VGA render */
2543 WREG32(VGA_RENDER_CONTROL, 0);
2545 /* blank the display controllers */
2546 for (i = 0; i < rdev->num_crtc; i++) {
2547 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2549 save->crtc_enabled[i] = true;
2550 if (ASIC_IS_DCE6(rdev)) {
2551 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2552 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2553 radeon_wait_for_vblank(rdev, i);
2554 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2555 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2556 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2559 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2560 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2561 radeon_wait_for_vblank(rdev, i);
2562 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2563 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2564 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2565 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2568 /* wait for the next frame */
2569 frame_count = radeon_get_vblank_counter(rdev, i);
2570 for (j = 0; j < rdev->usec_timeout; j++) {
2571 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2576 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2577 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2578 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2579 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2580 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2581 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2582 save->crtc_enabled[i] = false;
2585 save->crtc_enabled[i] = false;
2589 radeon_mc_wait_for_idle(rdev);
2591 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2592 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2593 /* Block CPU access */
2594 WREG32(BIF_FB_EN, 0);
2595 /* blackout the MC */
2596 blackout &= ~BLACKOUT_MODE_MASK;
2597 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2599 /* wait for the MC to settle */
2602 /* lock double buffered regs */
2603 for (i = 0; i < rdev->num_crtc; i++) {
2604 if (save->crtc_enabled[i]) {
2605 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2606 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2607 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2608 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2610 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2613 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2619 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2621 u32 tmp, frame_count;
2624 /* update crtc base addresses */
2625 for (i = 0; i < rdev->num_crtc; i++) {
2626 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2627 upper_32_bits(rdev->mc.vram_start));
2628 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2629 upper_32_bits(rdev->mc.vram_start));
2630 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2631 (u32)rdev->mc.vram_start);
2632 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2633 (u32)rdev->mc.vram_start);
2636 if (!ASIC_IS_NODCE(rdev)) {
2637 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2638 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2641 /* unlock regs and wait for update */
2642 for (i = 0; i < rdev->num_crtc; i++) {
2643 if (save->crtc_enabled[i]) {
2644 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2645 if ((tmp & 0x7) != 3) {
2648 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2650 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2651 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2652 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2653 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2655 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2658 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2660 for (j = 0; j < rdev->usec_timeout; j++) {
2661 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2662 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2669 /* unblackout the MC */
2670 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2671 tmp &= ~BLACKOUT_MODE_MASK;
2672 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2673 /* allow CPU access */
2674 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2676 for (i = 0; i < rdev->num_crtc; i++) {
2677 if (save->crtc_enabled[i]) {
2678 if (ASIC_IS_DCE6(rdev)) {
2679 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2680 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2681 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2682 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2683 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2685 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2686 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2687 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2688 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2689 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2691 /* wait for the next frame */
2692 frame_count = radeon_get_vblank_counter(rdev, i);
2693 for (j = 0; j < rdev->usec_timeout; j++) {
2694 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2700 if (!ASIC_IS_NODCE(rdev)) {
2701 /* Unlock vga access */
2702 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2704 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2708 void evergreen_mc_program(struct radeon_device *rdev)
2710 struct evergreen_mc_save save;
2714 /* Initialize HDP */
2715 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2716 WREG32((0x2c14 + j), 0x00000000);
2717 WREG32((0x2c18 + j), 0x00000000);
2718 WREG32((0x2c1c + j), 0x00000000);
2719 WREG32((0x2c20 + j), 0x00000000);
2720 WREG32((0x2c24 + j), 0x00000000);
2722 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2724 evergreen_mc_stop(rdev, &save);
2725 if (evergreen_mc_wait_for_idle(rdev)) {
2726 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2728 /* Lockout access through VGA aperture*/
2729 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2730 /* Update configuration */
2731 if (rdev->flags & RADEON_IS_AGP) {
2732 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2733 /* VRAM before AGP */
2734 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2735 rdev->mc.vram_start >> 12);
2736 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2737 rdev->mc.gtt_end >> 12);
2739 /* VRAM after AGP */
2740 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2741 rdev->mc.gtt_start >> 12);
2742 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2743 rdev->mc.vram_end >> 12);
2746 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2747 rdev->mc.vram_start >> 12);
2748 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2749 rdev->mc.vram_end >> 12);
2751 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2752 /* llano/ontario only */
2753 if ((rdev->family == CHIP_PALM) ||
2754 (rdev->family == CHIP_SUMO) ||
2755 (rdev->family == CHIP_SUMO2)) {
2756 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2757 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2758 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2759 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2761 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2762 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2763 WREG32(MC_VM_FB_LOCATION, tmp);
2764 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2765 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2766 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2767 if (rdev->flags & RADEON_IS_AGP) {
2768 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2769 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2770 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2772 WREG32(MC_VM_AGP_BASE, 0);
2773 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2774 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2776 if (evergreen_mc_wait_for_idle(rdev)) {
2777 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2779 evergreen_mc_resume(rdev, &save);
2780 /* we need to own VRAM, so turn off the VGA renderer here
2781 * to stop it overwriting our objects */
2782 rv515_vga_render_disable(rdev);
2788 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2790 struct radeon_ring *ring = &rdev->ring[ib->ring];
2793 /* set to DX10/11 mode */
2794 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2795 radeon_ring_write(ring, 1);
2797 if (ring->rptr_save_reg) {
2798 next_rptr = ring->wptr + 3 + 4;
2799 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2800 radeon_ring_write(ring, ((ring->rptr_save_reg -
2801 PACKET3_SET_CONFIG_REG_START) >> 2));
2802 radeon_ring_write(ring, next_rptr);
2803 } else if (rdev->wb.enabled) {
2804 next_rptr = ring->wptr + 5 + 4;
2805 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2806 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2807 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2808 radeon_ring_write(ring, next_rptr);
2809 radeon_ring_write(ring, 0);
2812 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2813 radeon_ring_write(ring,
2817 (ib->gpu_addr & 0xFFFFFFFC));
2818 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2819 radeon_ring_write(ring, ib->length_dw);
2823 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2825 const __be32 *fw_data;
2828 if (!rdev->me_fw || !rdev->pfp_fw)
2836 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2838 fw_data = (const __be32 *)rdev->pfp_fw->data;
2839 WREG32(CP_PFP_UCODE_ADDR, 0);
2840 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2841 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2842 WREG32(CP_PFP_UCODE_ADDR, 0);
2844 fw_data = (const __be32 *)rdev->me_fw->data;
2845 WREG32(CP_ME_RAM_WADDR, 0);
2846 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2847 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2849 WREG32(CP_PFP_UCODE_ADDR, 0);
2850 WREG32(CP_ME_RAM_WADDR, 0);
2851 WREG32(CP_ME_RAM_RADDR, 0);
2855 static int evergreen_cp_start(struct radeon_device *rdev)
2857 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2861 r = radeon_ring_lock(rdev, ring, 7);
2863 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2866 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2867 radeon_ring_write(ring, 0x1);
2868 radeon_ring_write(ring, 0x0);
2869 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2870 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2871 radeon_ring_write(ring, 0);
2872 radeon_ring_write(ring, 0);
2873 radeon_ring_unlock_commit(rdev, ring);
2876 WREG32(CP_ME_CNTL, cp_me);
2878 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2880 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2884 /* setup clear context state */
2885 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2886 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2888 for (i = 0; i < evergreen_default_size; i++)
2889 radeon_ring_write(ring, evergreen_default_state[i]);
2891 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2892 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2894 /* set clear context state */
2895 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2896 radeon_ring_write(ring, 0);
2898 /* SQ_VTX_BASE_VTX_LOC */
2899 radeon_ring_write(ring, 0xc0026f00);
2900 radeon_ring_write(ring, 0x00000000);
2901 radeon_ring_write(ring, 0x00000000);
2902 radeon_ring_write(ring, 0x00000000);
2905 radeon_ring_write(ring, 0xc0036f00);
2906 radeon_ring_write(ring, 0x00000bc4);
2907 radeon_ring_write(ring, 0xffffffff);
2908 radeon_ring_write(ring, 0xffffffff);
2909 radeon_ring_write(ring, 0xffffffff);
2911 radeon_ring_write(ring, 0xc0026900);
2912 radeon_ring_write(ring, 0x00000316);
2913 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2914 radeon_ring_write(ring, 0x00000010); /* */
2916 radeon_ring_unlock_commit(rdev, ring);
2921 static int evergreen_cp_resume(struct radeon_device *rdev)
2923 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2928 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2929 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2935 RREG32(GRBM_SOFT_RESET);
2937 WREG32(GRBM_SOFT_RESET, 0);
2938 RREG32(GRBM_SOFT_RESET);
2940 /* Set ring buffer size */
2941 rb_bufsz = order_base_2(ring->ring_size / 8);
2942 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2944 tmp |= BUF_SWAP_32BIT;
2946 WREG32(CP_RB_CNTL, tmp);
2947 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2948 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2950 /* Set the write pointer delay */
2951 WREG32(CP_RB_WPTR_DELAY, 0);
2953 /* Initialize the ring buffer's read and write pointers */
2954 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2955 WREG32(CP_RB_RPTR_WR, 0);
2957 WREG32(CP_RB_WPTR, ring->wptr);
2959 /* set the wb address whether it's enabled or not */
2960 WREG32(CP_RB_RPTR_ADDR,
2961 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2962 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2963 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2965 if (rdev->wb.enabled)
2966 WREG32(SCRATCH_UMSK, 0xff);
2968 tmp |= RB_NO_UPDATE;
2969 WREG32(SCRATCH_UMSK, 0);
2973 WREG32(CP_RB_CNTL, tmp);
2975 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2976 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2978 evergreen_cp_start(rdev);
2980 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2982 ring->ready = false;
2991 static void evergreen_gpu_init(struct radeon_device *rdev)
2994 u32 mc_shared_chmap, mc_arb_ramcfg;
2998 u32 sq_lds_resource_mgmt;
2999 u32 sq_gpr_resource_mgmt_1;
3000 u32 sq_gpr_resource_mgmt_2;
3001 u32 sq_gpr_resource_mgmt_3;
3002 u32 sq_thread_resource_mgmt;
3003 u32 sq_thread_resource_mgmt_2;
3004 u32 sq_stack_resource_mgmt_1;
3005 u32 sq_stack_resource_mgmt_2;
3006 u32 sq_stack_resource_mgmt_3;
3007 u32 vgt_cache_invalidation;
3008 u32 hdp_host_path_cntl, tmp;
3009 u32 disabled_rb_mask;
3010 int i, j, num_shader_engines, ps_thread_count;
3012 switch (rdev->family) {
3015 rdev->config.evergreen.num_ses = 2;
3016 rdev->config.evergreen.max_pipes = 4;
3017 rdev->config.evergreen.max_tile_pipes = 8;
3018 rdev->config.evergreen.max_simds = 10;
3019 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3020 rdev->config.evergreen.max_gprs = 256;
3021 rdev->config.evergreen.max_threads = 248;
3022 rdev->config.evergreen.max_gs_threads = 32;
3023 rdev->config.evergreen.max_stack_entries = 512;
3024 rdev->config.evergreen.sx_num_of_sets = 4;
3025 rdev->config.evergreen.sx_max_export_size = 256;
3026 rdev->config.evergreen.sx_max_export_pos_size = 64;
3027 rdev->config.evergreen.sx_max_export_smx_size = 192;
3028 rdev->config.evergreen.max_hw_contexts = 8;
3029 rdev->config.evergreen.sq_num_cf_insts = 2;
3031 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3032 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3033 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3034 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3037 rdev->config.evergreen.num_ses = 1;
3038 rdev->config.evergreen.max_pipes = 4;
3039 rdev->config.evergreen.max_tile_pipes = 4;
3040 rdev->config.evergreen.max_simds = 10;
3041 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3042 rdev->config.evergreen.max_gprs = 256;
3043 rdev->config.evergreen.max_threads = 248;
3044 rdev->config.evergreen.max_gs_threads = 32;
3045 rdev->config.evergreen.max_stack_entries = 512;
3046 rdev->config.evergreen.sx_num_of_sets = 4;
3047 rdev->config.evergreen.sx_max_export_size = 256;
3048 rdev->config.evergreen.sx_max_export_pos_size = 64;
3049 rdev->config.evergreen.sx_max_export_smx_size = 192;
3050 rdev->config.evergreen.max_hw_contexts = 8;
3051 rdev->config.evergreen.sq_num_cf_insts = 2;
3053 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3054 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3055 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3056 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3059 rdev->config.evergreen.num_ses = 1;
3060 rdev->config.evergreen.max_pipes = 4;
3061 rdev->config.evergreen.max_tile_pipes = 4;
3062 rdev->config.evergreen.max_simds = 5;
3063 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3064 rdev->config.evergreen.max_gprs = 256;
3065 rdev->config.evergreen.max_threads = 248;
3066 rdev->config.evergreen.max_gs_threads = 32;
3067 rdev->config.evergreen.max_stack_entries = 256;
3068 rdev->config.evergreen.sx_num_of_sets = 4;
3069 rdev->config.evergreen.sx_max_export_size = 256;
3070 rdev->config.evergreen.sx_max_export_pos_size = 64;
3071 rdev->config.evergreen.sx_max_export_smx_size = 192;
3072 rdev->config.evergreen.max_hw_contexts = 8;
3073 rdev->config.evergreen.sq_num_cf_insts = 2;
3075 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3076 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3077 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3078 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3082 rdev->config.evergreen.num_ses = 1;
3083 rdev->config.evergreen.max_pipes = 2;
3084 rdev->config.evergreen.max_tile_pipes = 2;
3085 rdev->config.evergreen.max_simds = 2;
3086 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3087 rdev->config.evergreen.max_gprs = 256;
3088 rdev->config.evergreen.max_threads = 192;
3089 rdev->config.evergreen.max_gs_threads = 16;
3090 rdev->config.evergreen.max_stack_entries = 256;
3091 rdev->config.evergreen.sx_num_of_sets = 4;
3092 rdev->config.evergreen.sx_max_export_size = 128;
3093 rdev->config.evergreen.sx_max_export_pos_size = 32;
3094 rdev->config.evergreen.sx_max_export_smx_size = 96;
3095 rdev->config.evergreen.max_hw_contexts = 4;
3096 rdev->config.evergreen.sq_num_cf_insts = 1;
3098 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3099 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3100 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3101 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3104 rdev->config.evergreen.num_ses = 1;
3105 rdev->config.evergreen.max_pipes = 2;
3106 rdev->config.evergreen.max_tile_pipes = 2;
3107 rdev->config.evergreen.max_simds = 2;
3108 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3109 rdev->config.evergreen.max_gprs = 256;
3110 rdev->config.evergreen.max_threads = 192;
3111 rdev->config.evergreen.max_gs_threads = 16;
3112 rdev->config.evergreen.max_stack_entries = 256;
3113 rdev->config.evergreen.sx_num_of_sets = 4;
3114 rdev->config.evergreen.sx_max_export_size = 128;
3115 rdev->config.evergreen.sx_max_export_pos_size = 32;
3116 rdev->config.evergreen.sx_max_export_smx_size = 96;
3117 rdev->config.evergreen.max_hw_contexts = 4;
3118 rdev->config.evergreen.sq_num_cf_insts = 1;
3120 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3121 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3122 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3123 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3126 rdev->config.evergreen.num_ses = 1;
3127 rdev->config.evergreen.max_pipes = 4;
3128 rdev->config.evergreen.max_tile_pipes = 4;
3129 if (rdev->pdev->device == 0x9648)
3130 rdev->config.evergreen.max_simds = 3;
3131 else if ((rdev->pdev->device == 0x9647) ||
3132 (rdev->pdev->device == 0x964a))
3133 rdev->config.evergreen.max_simds = 4;
3135 rdev->config.evergreen.max_simds = 5;
3136 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3137 rdev->config.evergreen.max_gprs = 256;
3138 rdev->config.evergreen.max_threads = 248;
3139 rdev->config.evergreen.max_gs_threads = 32;
3140 rdev->config.evergreen.max_stack_entries = 256;
3141 rdev->config.evergreen.sx_num_of_sets = 4;
3142 rdev->config.evergreen.sx_max_export_size = 256;
3143 rdev->config.evergreen.sx_max_export_pos_size = 64;
3144 rdev->config.evergreen.sx_max_export_smx_size = 192;
3145 rdev->config.evergreen.max_hw_contexts = 8;
3146 rdev->config.evergreen.sq_num_cf_insts = 2;
3148 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3149 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3150 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3151 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3154 rdev->config.evergreen.num_ses = 1;
3155 rdev->config.evergreen.max_pipes = 4;
3156 rdev->config.evergreen.max_tile_pipes = 4;
3157 rdev->config.evergreen.max_simds = 2;
3158 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3159 rdev->config.evergreen.max_gprs = 256;
3160 rdev->config.evergreen.max_threads = 248;
3161 rdev->config.evergreen.max_gs_threads = 32;
3162 rdev->config.evergreen.max_stack_entries = 512;
3163 rdev->config.evergreen.sx_num_of_sets = 4;
3164 rdev->config.evergreen.sx_max_export_size = 256;
3165 rdev->config.evergreen.sx_max_export_pos_size = 64;
3166 rdev->config.evergreen.sx_max_export_smx_size = 192;
3167 rdev->config.evergreen.max_hw_contexts = 4;
3168 rdev->config.evergreen.sq_num_cf_insts = 2;
3170 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3171 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3172 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3173 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3176 rdev->config.evergreen.num_ses = 2;
3177 rdev->config.evergreen.max_pipes = 4;
3178 rdev->config.evergreen.max_tile_pipes = 8;
3179 rdev->config.evergreen.max_simds = 7;
3180 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3181 rdev->config.evergreen.max_gprs = 256;
3182 rdev->config.evergreen.max_threads = 248;
3183 rdev->config.evergreen.max_gs_threads = 32;
3184 rdev->config.evergreen.max_stack_entries = 512;
3185 rdev->config.evergreen.sx_num_of_sets = 4;
3186 rdev->config.evergreen.sx_max_export_size = 256;
3187 rdev->config.evergreen.sx_max_export_pos_size = 64;
3188 rdev->config.evergreen.sx_max_export_smx_size = 192;
3189 rdev->config.evergreen.max_hw_contexts = 8;
3190 rdev->config.evergreen.sq_num_cf_insts = 2;
3192 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3193 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3194 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3195 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3198 rdev->config.evergreen.num_ses = 1;
3199 rdev->config.evergreen.max_pipes = 4;
3200 rdev->config.evergreen.max_tile_pipes = 4;
3201 rdev->config.evergreen.max_simds = 6;
3202 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3203 rdev->config.evergreen.max_gprs = 256;
3204 rdev->config.evergreen.max_threads = 248;
3205 rdev->config.evergreen.max_gs_threads = 32;
3206 rdev->config.evergreen.max_stack_entries = 256;
3207 rdev->config.evergreen.sx_num_of_sets = 4;
3208 rdev->config.evergreen.sx_max_export_size = 256;
3209 rdev->config.evergreen.sx_max_export_pos_size = 64;
3210 rdev->config.evergreen.sx_max_export_smx_size = 192;
3211 rdev->config.evergreen.max_hw_contexts = 8;
3212 rdev->config.evergreen.sq_num_cf_insts = 2;
3214 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3215 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3216 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3217 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3220 rdev->config.evergreen.num_ses = 1;
3221 rdev->config.evergreen.max_pipes = 2;
3222 rdev->config.evergreen.max_tile_pipes = 2;
3223 rdev->config.evergreen.max_simds = 2;
3224 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3225 rdev->config.evergreen.max_gprs = 256;
3226 rdev->config.evergreen.max_threads = 192;
3227 rdev->config.evergreen.max_gs_threads = 16;
3228 rdev->config.evergreen.max_stack_entries = 256;
3229 rdev->config.evergreen.sx_num_of_sets = 4;
3230 rdev->config.evergreen.sx_max_export_size = 128;
3231 rdev->config.evergreen.sx_max_export_pos_size = 32;
3232 rdev->config.evergreen.sx_max_export_smx_size = 96;
3233 rdev->config.evergreen.max_hw_contexts = 4;
3234 rdev->config.evergreen.sq_num_cf_insts = 1;
3236 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3237 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3238 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3239 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3243 /* Initialize HDP */
3244 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3245 WREG32((0x2c14 + j), 0x00000000);
3246 WREG32((0x2c18 + j), 0x00000000);
3247 WREG32((0x2c1c + j), 0x00000000);
3248 WREG32((0x2c20 + j), 0x00000000);
3249 WREG32((0x2c24 + j), 0x00000000);
3252 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3254 evergreen_fix_pci_max_read_req_size(rdev);
3256 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3257 if ((rdev->family == CHIP_PALM) ||
3258 (rdev->family == CHIP_SUMO) ||
3259 (rdev->family == CHIP_SUMO2))
3260 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3262 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3264 /* setup tiling info dword. gb_addr_config is not adequate since it does
3265 * not have bank info, so create a custom tiling dword.
3266 * bits 3:0 num_pipes
3267 * bits 7:4 num_banks
3268 * bits 11:8 group_size
3269 * bits 15:12 row_size
3271 rdev->config.evergreen.tile_config = 0;
3272 switch (rdev->config.evergreen.max_tile_pipes) {
3275 rdev->config.evergreen.tile_config |= (0 << 0);
3278 rdev->config.evergreen.tile_config |= (1 << 0);
3281 rdev->config.evergreen.tile_config |= (2 << 0);
3284 rdev->config.evergreen.tile_config |= (3 << 0);
3287 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3288 if (rdev->flags & RADEON_IS_IGP)
3289 rdev->config.evergreen.tile_config |= 1 << 4;
3291 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3292 case 0: /* four banks */
3293 rdev->config.evergreen.tile_config |= 0 << 4;
3295 case 1: /* eight banks */
3296 rdev->config.evergreen.tile_config |= 1 << 4;
3298 case 2: /* sixteen banks */
3300 rdev->config.evergreen.tile_config |= 2 << 4;
3304 rdev->config.evergreen.tile_config |= 0 << 8;
3305 rdev->config.evergreen.tile_config |=
3306 ((gb_addr_config & 0x30000000) >> 28) << 12;
3308 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3310 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3314 efuse_straps_4 = RREG32_RCU(0x204);
3315 efuse_straps_3 = RREG32_RCU(0x203);
3316 tmp = (((efuse_straps_4 & 0xf) << 4) |
3317 ((efuse_straps_3 & 0xf0000000) >> 28));
3320 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3321 u32 rb_disable_bitmap;
3323 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3324 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3325 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3327 tmp |= rb_disable_bitmap;
3330 /* enabled rb are just the one not disabled :) */
3331 disabled_rb_mask = tmp;
3333 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3335 /* if all the backends are disabled, fix it up here */
3336 if ((disabled_rb_mask & tmp) == tmp) {
3337 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3338 disabled_rb_mask &= ~(1 << i);
3341 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3342 u32 simd_disable_bitmap;
3344 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3345 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3346 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3347 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3349 tmp |= simd_disable_bitmap;
3351 rdev->config.evergreen.active_simds = hweight32(~tmp);
3353 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3354 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3356 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3357 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3358 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3359 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3360 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3361 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3362 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3364 if ((rdev->config.evergreen.max_backends == 1) &&
3365 (rdev->flags & RADEON_IS_IGP)) {
3366 if ((disabled_rb_mask & 3) == 1) {
3367 /* RB0 disabled, RB1 enabled */
3370 /* RB1 disabled, RB0 enabled */
3374 tmp = gb_addr_config & NUM_PIPES_MASK;
3375 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3376 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3378 WREG32(GB_BACKEND_MAP, tmp);
3380 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3381 WREG32(CGTS_TCC_DISABLE, 0);
3382 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3383 WREG32(CGTS_USER_TCC_DISABLE, 0);
3385 /* set HW defaults for 3D engine */
3386 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3387 ROQ_IB2_START(0x2b)));
3389 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3391 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3396 sx_debug_1 = RREG32(SX_DEBUG_1);
3397 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3398 WREG32(SX_DEBUG_1, sx_debug_1);
3401 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3402 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3403 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3404 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3406 if (rdev->family <= CHIP_SUMO2)
3407 WREG32(SMX_SAR_CTL0, 0x00010000);
3409 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3410 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3411 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3413 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3414 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3415 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3417 WREG32(VGT_NUM_INSTANCES, 1);
3418 WREG32(SPI_CONFIG_CNTL, 0);
3419 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3420 WREG32(CP_PERFMON_CNTL, 0);
3422 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3423 FETCH_FIFO_HIWATER(0x4) |
3424 DONE_FIFO_HIWATER(0xe0) |
3425 ALU_UPDATE_FIFO_HIWATER(0x8)));
3427 sq_config = RREG32(SQ_CONFIG);
3428 sq_config &= ~(PS_PRIO(3) |
3432 sq_config |= (VC_ENABLE |
3439 switch (rdev->family) {
3445 /* no vertex cache */
3446 sq_config &= ~VC_ENABLE;
3452 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3454 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3455 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3456 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3457 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3458 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3459 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3460 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3462 switch (rdev->family) {
3467 ps_thread_count = 96;
3470 ps_thread_count = 128;
3474 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3475 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3476 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3477 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3478 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3479 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3481 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3482 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3483 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3484 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3485 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3486 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3488 WREG32(SQ_CONFIG, sq_config);
3489 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3490 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3491 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3492 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3493 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3494 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3495 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3496 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3497 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3498 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3500 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3501 FORCE_EOV_MAX_REZ_CNT(255)));
3503 switch (rdev->family) {
3509 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3512 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3515 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3516 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3518 WREG32(VGT_GS_VERTEX_REUSE, 16);
3519 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3520 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3522 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3523 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3525 WREG32(CB_PERF_CTR0_SEL_0, 0);
3526 WREG32(CB_PERF_CTR0_SEL_1, 0);
3527 WREG32(CB_PERF_CTR1_SEL_0, 0);
3528 WREG32(CB_PERF_CTR1_SEL_1, 0);
3529 WREG32(CB_PERF_CTR2_SEL_0, 0);
3530 WREG32(CB_PERF_CTR2_SEL_1, 0);
3531 WREG32(CB_PERF_CTR3_SEL_0, 0);
3532 WREG32(CB_PERF_CTR3_SEL_1, 0);
3534 /* clear render buffer base addresses */
3535 WREG32(CB_COLOR0_BASE, 0);
3536 WREG32(CB_COLOR1_BASE, 0);
3537 WREG32(CB_COLOR2_BASE, 0);
3538 WREG32(CB_COLOR3_BASE, 0);
3539 WREG32(CB_COLOR4_BASE, 0);
3540 WREG32(CB_COLOR5_BASE, 0);
3541 WREG32(CB_COLOR6_BASE, 0);
3542 WREG32(CB_COLOR7_BASE, 0);
3543 WREG32(CB_COLOR8_BASE, 0);
3544 WREG32(CB_COLOR9_BASE, 0);
3545 WREG32(CB_COLOR10_BASE, 0);
3546 WREG32(CB_COLOR11_BASE, 0);
3548 /* set the shader const cache sizes to 0 */
3549 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3551 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3554 tmp = RREG32(HDP_MISC_CNTL);
3555 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3556 WREG32(HDP_MISC_CNTL, tmp);
3558 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3559 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3561 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3567 int evergreen_mc_init(struct radeon_device *rdev)
3570 int chansize, numchan;
3572 /* Get VRAM informations */
3573 rdev->mc.vram_is_ddr = true;
3574 if ((rdev->family == CHIP_PALM) ||
3575 (rdev->family == CHIP_SUMO) ||
3576 (rdev->family == CHIP_SUMO2))
3577 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3579 tmp = RREG32(MC_ARB_RAMCFG);
3580 if (tmp & CHANSIZE_OVERRIDE) {
3582 } else if (tmp & CHANSIZE_MASK) {
3587 tmp = RREG32(MC_SHARED_CHMAP);
3588 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3603 rdev->mc.vram_width = numchan * chansize;
3604 /* Could aper size report 0 ? */
3605 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3606 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3607 /* Setup GPU memory space */
3608 if ((rdev->family == CHIP_PALM) ||
3609 (rdev->family == CHIP_SUMO) ||
3610 (rdev->family == CHIP_SUMO2)) {
3611 /* size in bytes on fusion */
3612 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3613 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3615 /* size in MB on evergreen/cayman/tn */
3616 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3617 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3619 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3620 r700_vram_gtt_location(rdev, &rdev->mc);
3621 radeon_update_bandwidth_info(rdev);
3626 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3628 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3629 RREG32(GRBM_STATUS));
3630 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3631 RREG32(GRBM_STATUS_SE0));
3632 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3633 RREG32(GRBM_STATUS_SE1));
3634 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3635 RREG32(SRBM_STATUS));
3636 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3637 RREG32(SRBM_STATUS2));
3638 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3639 RREG32(CP_STALLED_STAT1));
3640 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3641 RREG32(CP_STALLED_STAT2));
3642 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3643 RREG32(CP_BUSY_STAT));
3644 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3646 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3647 RREG32(DMA_STATUS_REG));
3648 if (rdev->family >= CHIP_CAYMAN) {
3649 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3650 RREG32(DMA_STATUS_REG + 0x800));
3654 bool evergreen_is_display_hung(struct radeon_device *rdev)
3660 for (i = 0; i < rdev->num_crtc; i++) {
3661 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3662 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3663 crtc_hung |= (1 << i);
3667 for (j = 0; j < 10; j++) {
3668 for (i = 0; i < rdev->num_crtc; i++) {
3669 if (crtc_hung & (1 << i)) {
3670 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3671 if (tmp != crtc_status[i])
3672 crtc_hung &= ~(1 << i);
3683 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3689 tmp = RREG32(GRBM_STATUS);
3690 if (tmp & (PA_BUSY | SC_BUSY |
3692 TA_BUSY | VGT_BUSY |
3694 SPI_BUSY | VGT_BUSY_NO_DMA))
3695 reset_mask |= RADEON_RESET_GFX;
3697 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3698 CP_BUSY | CP_COHERENCY_BUSY))
3699 reset_mask |= RADEON_RESET_CP;
3701 if (tmp & GRBM_EE_BUSY)
3702 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3704 /* DMA_STATUS_REG */
3705 tmp = RREG32(DMA_STATUS_REG);
3706 if (!(tmp & DMA_IDLE))
3707 reset_mask |= RADEON_RESET_DMA;
3710 tmp = RREG32(SRBM_STATUS2);
3712 reset_mask |= RADEON_RESET_DMA;
3715 tmp = RREG32(SRBM_STATUS);
3716 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3717 reset_mask |= RADEON_RESET_RLC;
3720 reset_mask |= RADEON_RESET_IH;
3723 reset_mask |= RADEON_RESET_SEM;
3725 if (tmp & GRBM_RQ_PENDING)
3726 reset_mask |= RADEON_RESET_GRBM;
3729 reset_mask |= RADEON_RESET_VMC;
3731 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3732 MCC_BUSY | MCD_BUSY))
3733 reset_mask |= RADEON_RESET_MC;
3735 if (evergreen_is_display_hung(rdev))
3736 reset_mask |= RADEON_RESET_DISPLAY;
3739 tmp = RREG32(VM_L2_STATUS);
3741 reset_mask |= RADEON_RESET_VMC;
3743 /* Skip MC reset as it's mostly likely not hung, just busy */
3744 if (reset_mask & RADEON_RESET_MC) {
3745 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3746 reset_mask &= ~RADEON_RESET_MC;
3752 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3754 struct evergreen_mc_save save;
3755 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3758 if (reset_mask == 0)
3761 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3763 evergreen_print_gpu_status_regs(rdev);
3765 /* Disable CP parsing/prefetching */
3766 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3768 if (reset_mask & RADEON_RESET_DMA) {
3770 tmp = RREG32(DMA_RB_CNTL);
3771 tmp &= ~DMA_RB_ENABLE;
3772 WREG32(DMA_RB_CNTL, tmp);
3777 evergreen_mc_stop(rdev, &save);
3778 if (evergreen_mc_wait_for_idle(rdev)) {
3779 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3782 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3783 grbm_soft_reset |= SOFT_RESET_DB |
3796 if (reset_mask & RADEON_RESET_CP) {
3797 grbm_soft_reset |= SOFT_RESET_CP |
3800 srbm_soft_reset |= SOFT_RESET_GRBM;
3803 if (reset_mask & RADEON_RESET_DMA)
3804 srbm_soft_reset |= SOFT_RESET_DMA;
3806 if (reset_mask & RADEON_RESET_DISPLAY)
3807 srbm_soft_reset |= SOFT_RESET_DC;
3809 if (reset_mask & RADEON_RESET_RLC)
3810 srbm_soft_reset |= SOFT_RESET_RLC;
3812 if (reset_mask & RADEON_RESET_SEM)
3813 srbm_soft_reset |= SOFT_RESET_SEM;
3815 if (reset_mask & RADEON_RESET_IH)
3816 srbm_soft_reset |= SOFT_RESET_IH;
3818 if (reset_mask & RADEON_RESET_GRBM)
3819 srbm_soft_reset |= SOFT_RESET_GRBM;
3821 if (reset_mask & RADEON_RESET_VMC)
3822 srbm_soft_reset |= SOFT_RESET_VMC;
3824 if (!(rdev->flags & RADEON_IS_IGP)) {
3825 if (reset_mask & RADEON_RESET_MC)
3826 srbm_soft_reset |= SOFT_RESET_MC;
3829 if (grbm_soft_reset) {
3830 tmp = RREG32(GRBM_SOFT_RESET);
3831 tmp |= grbm_soft_reset;
3832 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3833 WREG32(GRBM_SOFT_RESET, tmp);
3834 tmp = RREG32(GRBM_SOFT_RESET);
3838 tmp &= ~grbm_soft_reset;
3839 WREG32(GRBM_SOFT_RESET, tmp);
3840 tmp = RREG32(GRBM_SOFT_RESET);
3843 if (srbm_soft_reset) {
3844 tmp = RREG32(SRBM_SOFT_RESET);
3845 tmp |= srbm_soft_reset;
3846 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3847 WREG32(SRBM_SOFT_RESET, tmp);
3848 tmp = RREG32(SRBM_SOFT_RESET);
3852 tmp &= ~srbm_soft_reset;
3853 WREG32(SRBM_SOFT_RESET, tmp);
3854 tmp = RREG32(SRBM_SOFT_RESET);
3857 /* Wait a little for things to settle down */
3860 evergreen_mc_resume(rdev, &save);
3863 evergreen_print_gpu_status_regs(rdev);
3866 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3868 struct evergreen_mc_save save;
3871 dev_info(rdev->dev, "GPU pci config reset\n");
3875 /* Disable CP parsing/prefetching */
3876 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3879 tmp = RREG32(DMA_RB_CNTL);
3880 tmp &= ~DMA_RB_ENABLE;
3881 WREG32(DMA_RB_CNTL, tmp);
3882 /* XXX other engines? */
3885 r600_rlc_stop(rdev);
3889 /* set mclk/sclk to bypass */
3890 rv770_set_clk_bypass_mode(rdev);
3892 pci_clear_master(rdev->pdev);
3893 /* disable mem access */
3894 evergreen_mc_stop(rdev, &save);
3895 if (evergreen_mc_wait_for_idle(rdev)) {
3896 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3899 radeon_pci_config_reset(rdev);
3900 /* wait for asic to come out of reset */
3901 for (i = 0; i < rdev->usec_timeout; i++) {
3902 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3908 int evergreen_asic_reset(struct radeon_device *rdev)
3912 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3915 r600_set_bios_scratch_engine_hung(rdev, true);
3917 /* try soft reset */
3918 evergreen_gpu_soft_reset(rdev, reset_mask);
3920 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3922 /* try pci config reset */
3923 if (reset_mask && radeon_hard_reset)
3924 evergreen_gpu_pci_config_reset(rdev);
3926 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3929 r600_set_bios_scratch_engine_hung(rdev, false);
3935 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3937 * @rdev: radeon_device pointer
3938 * @ring: radeon_ring structure holding ring information
3940 * Check if the GFX engine is locked up.
3941 * Returns true if the engine appears to be locked up, false if not.
3943 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3945 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3947 if (!(reset_mask & (RADEON_RESET_GFX |
3948 RADEON_RESET_COMPUTE |
3949 RADEON_RESET_CP))) {
3950 radeon_ring_lockup_update(rdev, ring);
3953 return radeon_ring_test_lockup(rdev, ring);
3959 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3960 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
3962 void sumo_rlc_fini(struct radeon_device *rdev)
3966 /* save restore block */
3967 if (rdev->rlc.save_restore_obj) {
3968 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3969 if (unlikely(r != 0))
3970 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3971 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3972 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3974 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3975 rdev->rlc.save_restore_obj = NULL;
3978 /* clear state block */
3979 if (rdev->rlc.clear_state_obj) {
3980 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3981 if (unlikely(r != 0))
3982 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3983 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3984 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3986 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3987 rdev->rlc.clear_state_obj = NULL;
3990 /* clear state block */
3991 if (rdev->rlc.cp_table_obj) {
3992 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3993 if (unlikely(r != 0))
3994 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3995 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3996 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3998 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3999 rdev->rlc.cp_table_obj = NULL;
4003 #define CP_ME_TABLE_SIZE 96
4005 int sumo_rlc_init(struct radeon_device *rdev)
4008 volatile u32 *dst_ptr;
4009 u32 dws, data, i, j, k, reg_num;
4010 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4011 u64 reg_list_mc_addr;
4012 const struct cs_section_def *cs_data;
4015 src_ptr = rdev->rlc.reg_list;
4016 dws = rdev->rlc.reg_list_size;
4017 if (rdev->family >= CHIP_BONAIRE) {
4018 dws += (5 * 16) + 48 + 48 + 64;
4020 cs_data = rdev->rlc.cs_data;
4023 /* save restore block */
4024 if (rdev->rlc.save_restore_obj == NULL) {
4025 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4026 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
4028 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4033 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4034 if (unlikely(r != 0)) {
4035 sumo_rlc_fini(rdev);
4038 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4039 &rdev->rlc.save_restore_gpu_addr);
4041 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4042 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4043 sumo_rlc_fini(rdev);
4047 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4049 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4050 sumo_rlc_fini(rdev);
4053 /* write the sr buffer */
4054 dst_ptr = rdev->rlc.sr_ptr;
4055 if (rdev->family >= CHIP_TAHITI) {
4057 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4058 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4062 * dw0: (reg2 << 16) | reg1
4063 * dw1: reg1 save space
4064 * dw2: reg2 save space
4066 for (i = 0; i < dws; i++) {
4067 data = src_ptr[i] >> 2;
4070 data |= (src_ptr[i] >> 2) << 16;
4071 j = (((i - 1) * 3) / 2);
4072 dst_ptr[j] = cpu_to_le32(data);
4075 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4077 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4078 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4082 /* clear state block */
4083 if (rdev->family >= CHIP_BONAIRE) {
4084 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4085 } else if (rdev->family >= CHIP_TAHITI) {
4086 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4087 dws = rdev->rlc.clear_state_size + (256 / 4);
4091 for (i = 0; cs_data[i].section != NULL; i++) {
4092 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4094 dws += cs_data[i].section[j].reg_count;
4097 reg_list_blk_index = (3 * reg_list_num + 2);
4098 dws += reg_list_blk_index;
4099 rdev->rlc.clear_state_size = dws;
4102 if (rdev->rlc.clear_state_obj == NULL) {
4103 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4104 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4106 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4107 sumo_rlc_fini(rdev);
4111 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4112 if (unlikely(r != 0)) {
4113 sumo_rlc_fini(rdev);
4116 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4117 &rdev->rlc.clear_state_gpu_addr);
4119 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4120 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4121 sumo_rlc_fini(rdev);
4125 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4127 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4128 sumo_rlc_fini(rdev);
4131 /* set up the cs buffer */
4132 dst_ptr = rdev->rlc.cs_ptr;
4133 if (rdev->family >= CHIP_BONAIRE) {
4134 cik_get_csb_buffer(rdev, dst_ptr);
4135 } else if (rdev->family >= CHIP_TAHITI) {
4136 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4137 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4138 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4139 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4140 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4142 reg_list_hdr_blk_index = 0;
4143 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4144 data = upper_32_bits(reg_list_mc_addr);
4145 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4146 reg_list_hdr_blk_index++;
4147 for (i = 0; cs_data[i].section != NULL; i++) {
4148 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4149 reg_num = cs_data[i].section[j].reg_count;
4150 data = reg_list_mc_addr & 0xffffffff;
4151 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4152 reg_list_hdr_blk_index++;
4154 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4155 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4156 reg_list_hdr_blk_index++;
4158 data = 0x08000000 | (reg_num * 4);
4159 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4160 reg_list_hdr_blk_index++;
4162 for (k = 0; k < reg_num; k++) {
4163 data = cs_data[i].section[j].extent[k];
4164 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4166 reg_list_mc_addr += reg_num * 4;
4167 reg_list_blk_index += reg_num;
4170 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4172 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4173 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4176 if (rdev->rlc.cp_table_size) {
4177 if (rdev->rlc.cp_table_obj == NULL) {
4178 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true,
4179 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj);
4181 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4182 sumo_rlc_fini(rdev);
4187 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4188 if (unlikely(r != 0)) {
4189 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4190 sumo_rlc_fini(rdev);
4193 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4194 &rdev->rlc.cp_table_gpu_addr);
4196 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4197 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4198 sumo_rlc_fini(rdev);
4201 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4203 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4204 sumo_rlc_fini(rdev);
4208 cik_init_cp_pg_table(rdev);
4210 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4211 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4218 static void evergreen_rlc_start(struct radeon_device *rdev)
4220 u32 mask = RLC_ENABLE;
4222 if (rdev->flags & RADEON_IS_IGP) {
4223 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4226 WREG32(RLC_CNTL, mask);
4229 int evergreen_rlc_resume(struct radeon_device *rdev)
4232 const __be32 *fw_data;
4237 r600_rlc_stop(rdev);
4239 WREG32(RLC_HB_CNTL, 0);
4241 if (rdev->flags & RADEON_IS_IGP) {
4242 if (rdev->family == CHIP_ARUBA) {
4243 u32 always_on_bitmap =
4244 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4245 /* find out the number of active simds */
4246 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4247 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4248 tmp = hweight32(~tmp);
4249 if (tmp == rdev->config.cayman.max_simds_per_se) {
4250 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4251 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4252 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4253 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4254 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4257 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4258 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4260 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4261 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4263 WREG32(RLC_HB_BASE, 0);
4264 WREG32(RLC_HB_RPTR, 0);
4265 WREG32(RLC_HB_WPTR, 0);
4266 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4267 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4269 WREG32(RLC_MC_CNTL, 0);
4270 WREG32(RLC_UCODE_CNTL, 0);
4272 fw_data = (const __be32 *)rdev->rlc_fw->data;
4273 if (rdev->family >= CHIP_ARUBA) {
4274 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4275 WREG32(RLC_UCODE_ADDR, i);
4276 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4278 } else if (rdev->family >= CHIP_CAYMAN) {
4279 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4280 WREG32(RLC_UCODE_ADDR, i);
4281 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4284 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4285 WREG32(RLC_UCODE_ADDR, i);
4286 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4289 WREG32(RLC_UCODE_ADDR, 0);
4291 evergreen_rlc_start(rdev);
4298 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4300 if (crtc >= rdev->num_crtc)
4303 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4306 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4310 if (rdev->family >= CHIP_CAYMAN) {
4311 cayman_cp_int_cntl_setup(rdev, 0,
4312 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4313 cayman_cp_int_cntl_setup(rdev, 1, 0);
4314 cayman_cp_int_cntl_setup(rdev, 2, 0);
4315 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4316 WREG32(CAYMAN_DMA1_CNTL, tmp);
4318 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4319 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4320 WREG32(DMA_CNTL, tmp);
4321 WREG32(GRBM_INT_CNTL, 0);
4322 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4323 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4324 if (rdev->num_crtc >= 4) {
4325 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4326 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4328 if (rdev->num_crtc >= 6) {
4329 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4330 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4333 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4334 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4335 if (rdev->num_crtc >= 4) {
4336 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4337 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4339 if (rdev->num_crtc >= 6) {
4340 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4341 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4344 /* only one DAC on DCE5 */
4345 if (!ASIC_IS_DCE5(rdev))
4346 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4347 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4349 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4350 WREG32(DC_HPD1_INT_CONTROL, tmp);
4351 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4352 WREG32(DC_HPD2_INT_CONTROL, tmp);
4353 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4354 WREG32(DC_HPD3_INT_CONTROL, tmp);
4355 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4356 WREG32(DC_HPD4_INT_CONTROL, tmp);
4357 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4358 WREG32(DC_HPD5_INT_CONTROL, tmp);
4359 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4360 WREG32(DC_HPD6_INT_CONTROL, tmp);
4364 int evergreen_irq_set(struct radeon_device *rdev)
4366 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4367 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4368 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4369 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4370 u32 grbm_int_cntl = 0;
4371 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4372 u32 dma_cntl, dma_cntl1 = 0;
4373 u32 thermal_int = 0;
4375 if (!rdev->irq.installed) {
4376 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4379 /* don't enable anything if the ih is disabled */
4380 if (!rdev->ih.enabled) {
4381 r600_disable_interrupts(rdev);
4382 /* force the active interrupt state to all disabled */
4383 evergreen_disable_interrupt_state(rdev);
4387 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4388 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4389 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4390 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4391 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4392 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4393 if (rdev->family == CHIP_ARUBA)
4394 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4395 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4397 thermal_int = RREG32(CG_THERMAL_INT) &
4398 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4400 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4401 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4402 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4403 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4404 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4405 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4407 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4409 if (rdev->family >= CHIP_CAYMAN) {
4410 /* enable CP interrupts on all rings */
4411 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4412 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4413 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4415 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4416 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4417 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4419 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4420 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4421 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4424 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4425 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4426 cp_int_cntl |= RB_INT_ENABLE;
4427 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4431 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4432 DRM_DEBUG("r600_irq_set: sw int dma\n");
4433 dma_cntl |= TRAP_ENABLE;
4436 if (rdev->family >= CHIP_CAYMAN) {
4437 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4438 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4439 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4440 dma_cntl1 |= TRAP_ENABLE;
4444 if (rdev->irq.dpm_thermal) {
4445 DRM_DEBUG("dpm thermal\n");
4446 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4449 if (rdev->irq.crtc_vblank_int[0] ||
4450 atomic_read(&rdev->irq.pflip[0])) {
4451 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4452 crtc1 |= VBLANK_INT_MASK;
4454 if (rdev->irq.crtc_vblank_int[1] ||
4455 atomic_read(&rdev->irq.pflip[1])) {
4456 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4457 crtc2 |= VBLANK_INT_MASK;
4459 if (rdev->irq.crtc_vblank_int[2] ||
4460 atomic_read(&rdev->irq.pflip[2])) {
4461 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4462 crtc3 |= VBLANK_INT_MASK;
4464 if (rdev->irq.crtc_vblank_int[3] ||
4465 atomic_read(&rdev->irq.pflip[3])) {
4466 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4467 crtc4 |= VBLANK_INT_MASK;
4469 if (rdev->irq.crtc_vblank_int[4] ||
4470 atomic_read(&rdev->irq.pflip[4])) {
4471 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4472 crtc5 |= VBLANK_INT_MASK;
4474 if (rdev->irq.crtc_vblank_int[5] ||
4475 atomic_read(&rdev->irq.pflip[5])) {
4476 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4477 crtc6 |= VBLANK_INT_MASK;
4479 if (rdev->irq.hpd[0]) {
4480 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4481 hpd1 |= DC_HPDx_INT_EN;
4483 if (rdev->irq.hpd[1]) {
4484 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4485 hpd2 |= DC_HPDx_INT_EN;
4487 if (rdev->irq.hpd[2]) {
4488 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4489 hpd3 |= DC_HPDx_INT_EN;
4491 if (rdev->irq.hpd[3]) {
4492 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4493 hpd4 |= DC_HPDx_INT_EN;
4495 if (rdev->irq.hpd[4]) {
4496 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4497 hpd5 |= DC_HPDx_INT_EN;
4499 if (rdev->irq.hpd[5]) {
4500 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4501 hpd6 |= DC_HPDx_INT_EN;
4503 if (rdev->irq.afmt[0]) {
4504 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4505 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4507 if (rdev->irq.afmt[1]) {
4508 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4509 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4511 if (rdev->irq.afmt[2]) {
4512 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4513 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4515 if (rdev->irq.afmt[3]) {
4516 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4517 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4519 if (rdev->irq.afmt[4]) {
4520 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4521 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4523 if (rdev->irq.afmt[5]) {
4524 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4525 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4528 if (rdev->family >= CHIP_CAYMAN) {
4529 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4530 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4531 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4533 WREG32(CP_INT_CNTL, cp_int_cntl);
4535 WREG32(DMA_CNTL, dma_cntl);
4537 if (rdev->family >= CHIP_CAYMAN)
4538 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4540 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4542 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4543 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4544 if (rdev->num_crtc >= 4) {
4545 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4546 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4548 if (rdev->num_crtc >= 6) {
4549 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4550 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4553 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4554 GRPH_PFLIP_INT_MASK);
4555 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4556 GRPH_PFLIP_INT_MASK);
4557 if (rdev->num_crtc >= 4) {
4558 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4559 GRPH_PFLIP_INT_MASK);
4560 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4561 GRPH_PFLIP_INT_MASK);
4563 if (rdev->num_crtc >= 6) {
4564 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4565 GRPH_PFLIP_INT_MASK);
4566 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4567 GRPH_PFLIP_INT_MASK);
4570 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4571 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4572 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4573 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4574 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4575 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4576 if (rdev->family == CHIP_ARUBA)
4577 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4579 WREG32(CG_THERMAL_INT, thermal_int);
4581 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4582 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4583 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4584 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4585 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4586 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4591 static void evergreen_irq_ack(struct radeon_device *rdev)
4595 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4596 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4597 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4598 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4599 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4600 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4601 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4602 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4603 if (rdev->num_crtc >= 4) {
4604 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4605 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4607 if (rdev->num_crtc >= 6) {
4608 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4609 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4612 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4613 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4614 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4615 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4616 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4617 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4619 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4620 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4621 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4622 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4623 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4624 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4625 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4626 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4627 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4628 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4629 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4630 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4632 if (rdev->num_crtc >= 4) {
4633 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4634 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4635 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4636 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4637 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4638 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4639 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4640 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4641 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4642 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4643 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4644 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4647 if (rdev->num_crtc >= 6) {
4648 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4649 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4650 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4651 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4652 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4653 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4654 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4655 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4656 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4657 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4658 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4659 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4662 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4663 tmp = RREG32(DC_HPD1_INT_CONTROL);
4664 tmp |= DC_HPDx_INT_ACK;
4665 WREG32(DC_HPD1_INT_CONTROL, tmp);
4667 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4668 tmp = RREG32(DC_HPD2_INT_CONTROL);
4669 tmp |= DC_HPDx_INT_ACK;
4670 WREG32(DC_HPD2_INT_CONTROL, tmp);
4672 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4673 tmp = RREG32(DC_HPD3_INT_CONTROL);
4674 tmp |= DC_HPDx_INT_ACK;
4675 WREG32(DC_HPD3_INT_CONTROL, tmp);
4677 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4678 tmp = RREG32(DC_HPD4_INT_CONTROL);
4679 tmp |= DC_HPDx_INT_ACK;
4680 WREG32(DC_HPD4_INT_CONTROL, tmp);
4682 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4683 tmp = RREG32(DC_HPD5_INT_CONTROL);
4684 tmp |= DC_HPDx_INT_ACK;
4685 WREG32(DC_HPD5_INT_CONTROL, tmp);
4687 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4688 tmp = RREG32(DC_HPD5_INT_CONTROL);
4689 tmp |= DC_HPDx_INT_ACK;
4690 WREG32(DC_HPD6_INT_CONTROL, tmp);
4692 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4693 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4694 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4695 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4697 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4698 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4699 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4700 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4702 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4703 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4704 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4705 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4707 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4708 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4709 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4710 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4712 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4713 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4714 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4715 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4717 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4718 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4719 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4720 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4724 static void evergreen_irq_disable(struct radeon_device *rdev)
4726 r600_disable_interrupts(rdev);
4727 /* Wait and acknowledge irq */
4729 evergreen_irq_ack(rdev);
4730 evergreen_disable_interrupt_state(rdev);
4733 void evergreen_irq_suspend(struct radeon_device *rdev)
4735 evergreen_irq_disable(rdev);
4736 r600_rlc_stop(rdev);
4739 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4743 if (rdev->wb.enabled)
4744 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4746 wptr = RREG32(IH_RB_WPTR);
4748 if (wptr & RB_OVERFLOW) {
4749 /* When a ring buffer overflow happen start parsing interrupt
4750 * from the last not overwritten vector (wptr + 16). Hopefully
4751 * this should allow us to catchup.
4753 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4754 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4755 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4756 tmp = RREG32(IH_RB_CNTL);
4757 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4758 WREG32(IH_RB_CNTL, tmp);
4760 return (wptr & rdev->ih.ptr_mask);
4763 int evergreen_irq_process(struct radeon_device *rdev)
4767 u32 src_id, src_data;
4769 bool queue_hotplug = false;
4770 bool queue_hdmi = false;
4771 bool queue_thermal = false;
4774 if (!rdev->ih.enabled || rdev->shutdown)
4777 wptr = evergreen_get_ih_wptr(rdev);
4780 /* is somebody else already processing irqs? */
4781 if (atomic_xchg(&rdev->ih.lock, 1))
4784 rptr = rdev->ih.rptr;
4785 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4787 /* Order reading of wptr vs. reading of IH ring data */
4790 /* display interrupts */
4791 evergreen_irq_ack(rdev);
4793 while (rptr != wptr) {
4794 /* wptr/rptr are in bytes! */
4795 ring_index = rptr / 4;
4796 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4797 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4800 case 1: /* D1 vblank/vline */
4802 case 0: /* D1 vblank */
4803 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4804 if (rdev->irq.crtc_vblank_int[0]) {
4805 drm_handle_vblank(rdev->ddev, 0);
4806 rdev->pm.vblank_sync = true;
4807 wake_up(&rdev->irq.vblank_queue);
4809 if (atomic_read(&rdev->irq.pflip[0]))
4810 radeon_crtc_handle_vblank(rdev, 0);
4811 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4812 DRM_DEBUG("IH: D1 vblank\n");
4815 case 1: /* D1 vline */
4816 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4817 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4818 DRM_DEBUG("IH: D1 vline\n");
4822 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4826 case 2: /* D2 vblank/vline */
4828 case 0: /* D2 vblank */
4829 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4830 if (rdev->irq.crtc_vblank_int[1]) {
4831 drm_handle_vblank(rdev->ddev, 1);
4832 rdev->pm.vblank_sync = true;
4833 wake_up(&rdev->irq.vblank_queue);
4835 if (atomic_read(&rdev->irq.pflip[1]))
4836 radeon_crtc_handle_vblank(rdev, 1);
4837 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4838 DRM_DEBUG("IH: D2 vblank\n");
4841 case 1: /* D2 vline */
4842 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4843 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4844 DRM_DEBUG("IH: D2 vline\n");
4848 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4852 case 3: /* D3 vblank/vline */
4854 case 0: /* D3 vblank */
4855 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4856 if (rdev->irq.crtc_vblank_int[2]) {
4857 drm_handle_vblank(rdev->ddev, 2);
4858 rdev->pm.vblank_sync = true;
4859 wake_up(&rdev->irq.vblank_queue);
4861 if (atomic_read(&rdev->irq.pflip[2]))
4862 radeon_crtc_handle_vblank(rdev, 2);
4863 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4864 DRM_DEBUG("IH: D3 vblank\n");
4867 case 1: /* D3 vline */
4868 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4869 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4870 DRM_DEBUG("IH: D3 vline\n");
4874 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4878 case 4: /* D4 vblank/vline */
4880 case 0: /* D4 vblank */
4881 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4882 if (rdev->irq.crtc_vblank_int[3]) {
4883 drm_handle_vblank(rdev->ddev, 3);
4884 rdev->pm.vblank_sync = true;
4885 wake_up(&rdev->irq.vblank_queue);
4887 if (atomic_read(&rdev->irq.pflip[3]))
4888 radeon_crtc_handle_vblank(rdev, 3);
4889 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4890 DRM_DEBUG("IH: D4 vblank\n");
4893 case 1: /* D4 vline */
4894 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4895 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4896 DRM_DEBUG("IH: D4 vline\n");
4900 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4904 case 5: /* D5 vblank/vline */
4906 case 0: /* D5 vblank */
4907 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4908 if (rdev->irq.crtc_vblank_int[4]) {
4909 drm_handle_vblank(rdev->ddev, 4);
4910 rdev->pm.vblank_sync = true;
4911 wake_up(&rdev->irq.vblank_queue);
4913 if (atomic_read(&rdev->irq.pflip[4]))
4914 radeon_crtc_handle_vblank(rdev, 4);
4915 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4916 DRM_DEBUG("IH: D5 vblank\n");
4919 case 1: /* D5 vline */
4920 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4921 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4922 DRM_DEBUG("IH: D5 vline\n");
4926 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4930 case 6: /* D6 vblank/vline */
4932 case 0: /* D6 vblank */
4933 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4934 if (rdev->irq.crtc_vblank_int[5]) {
4935 drm_handle_vblank(rdev->ddev, 5);
4936 rdev->pm.vblank_sync = true;
4937 wake_up(&rdev->irq.vblank_queue);
4939 if (atomic_read(&rdev->irq.pflip[5]))
4940 radeon_crtc_handle_vblank(rdev, 5);
4941 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4942 DRM_DEBUG("IH: D6 vblank\n");
4945 case 1: /* D6 vline */
4946 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4947 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4948 DRM_DEBUG("IH: D6 vline\n");
4952 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4956 case 8: /* D1 page flip */
4957 case 10: /* D2 page flip */
4958 case 12: /* D3 page flip */
4959 case 14: /* D4 page flip */
4960 case 16: /* D5 page flip */
4961 case 18: /* D6 page flip */
4962 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4963 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4965 case 42: /* HPD hotplug */
4968 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4969 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4970 queue_hotplug = true;
4971 DRM_DEBUG("IH: HPD1\n");
4975 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4976 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4977 queue_hotplug = true;
4978 DRM_DEBUG("IH: HPD2\n");
4982 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4983 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4984 queue_hotplug = true;
4985 DRM_DEBUG("IH: HPD3\n");
4989 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4990 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4991 queue_hotplug = true;
4992 DRM_DEBUG("IH: HPD4\n");
4996 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4997 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4998 queue_hotplug = true;
4999 DRM_DEBUG("IH: HPD5\n");
5003 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5004 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5005 queue_hotplug = true;
5006 DRM_DEBUG("IH: HPD6\n");
5010 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5017 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
5018 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5020 DRM_DEBUG("IH: HDMI0\n");
5024 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
5025 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5027 DRM_DEBUG("IH: HDMI1\n");
5031 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
5032 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5034 DRM_DEBUG("IH: HDMI2\n");
5038 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
5039 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5041 DRM_DEBUG("IH: HDMI3\n");
5045 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
5046 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5048 DRM_DEBUG("IH: HDMI4\n");
5052 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5053 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5055 DRM_DEBUG("IH: HDMI5\n");
5059 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5063 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5064 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5068 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5069 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5070 /* reset addr and status */
5071 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5072 if (addr == 0x0 && status == 0x0)
5074 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5075 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5077 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5079 cayman_vm_decode_fault(rdev, status, addr);
5081 case 176: /* CP_INT in ring buffer */
5082 case 177: /* CP_INT in IB1 */
5083 case 178: /* CP_INT in IB2 */
5084 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5085 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5087 case 181: /* CP EOP event */
5088 DRM_DEBUG("IH: CP EOP\n");
5089 if (rdev->family >= CHIP_CAYMAN) {
5092 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5095 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5098 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5102 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5104 case 224: /* DMA trap event */
5105 DRM_DEBUG("IH: DMA trap\n");
5106 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5108 case 230: /* thermal low to high */
5109 DRM_DEBUG("IH: thermal low to high\n");
5110 rdev->pm.dpm.thermal.high_to_low = false;
5111 queue_thermal = true;
5113 case 231: /* thermal high to low */
5114 DRM_DEBUG("IH: thermal high to low\n");
5115 rdev->pm.dpm.thermal.high_to_low = true;
5116 queue_thermal = true;
5118 case 233: /* GUI IDLE */
5119 DRM_DEBUG("IH: GUI idle\n");
5121 case 244: /* DMA trap event */
5122 if (rdev->family >= CHIP_CAYMAN) {
5123 DRM_DEBUG("IH: DMA1 trap\n");
5124 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5128 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5132 /* wptr/rptr are in bytes! */
5134 rptr &= rdev->ih.ptr_mask;
5137 schedule_work(&rdev->hotplug_work);
5139 schedule_work(&rdev->audio_work);
5140 if (queue_thermal && rdev->pm.dpm_enabled)
5141 schedule_work(&rdev->pm.dpm.thermal.work);
5142 rdev->ih.rptr = rptr;
5143 WREG32(IH_RB_RPTR, rdev->ih.rptr);
5144 atomic_set(&rdev->ih.lock, 0);
5146 /* make sure wptr hasn't changed while processing */
5147 wptr = evergreen_get_ih_wptr(rdev);
5154 static int evergreen_startup(struct radeon_device *rdev)
5156 struct radeon_ring *ring;
5159 /* enable pcie gen2 link */
5160 evergreen_pcie_gen2_enable(rdev);
5162 evergreen_program_aspm(rdev);
5164 /* scratch needs to be initialized before MC */
5165 r = r600_vram_scratch_init(rdev);
5169 evergreen_mc_program(rdev);
5171 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5172 r = ni_mc_load_microcode(rdev);
5174 DRM_ERROR("Failed to load MC firmware!\n");
5179 if (rdev->flags & RADEON_IS_AGP) {
5180 evergreen_agp_enable(rdev);
5182 r = evergreen_pcie_gart_enable(rdev);
5186 evergreen_gpu_init(rdev);
5188 /* allocate rlc buffers */
5189 if (rdev->flags & RADEON_IS_IGP) {
5190 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5191 rdev->rlc.reg_list_size =
5192 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5193 rdev->rlc.cs_data = evergreen_cs_data;
5194 r = sumo_rlc_init(rdev);
5196 DRM_ERROR("Failed to init rlc BOs!\n");
5201 /* allocate wb buffer */
5202 r = radeon_wb_init(rdev);
5206 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5208 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5212 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5214 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5218 r = uvd_v2_2_resume(rdev);
5220 r = radeon_fence_driver_start_ring(rdev,
5221 R600_RING_TYPE_UVD_INDEX);
5223 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5227 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5230 if (!rdev->irq.installed) {
5231 r = radeon_irq_kms_init(rdev);
5236 r = r600_irq_init(rdev);
5238 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5239 radeon_irq_kms_fini(rdev);
5242 evergreen_irq_set(rdev);
5244 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5245 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5250 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5251 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5252 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5256 r = evergreen_cp_load_microcode(rdev);
5259 r = evergreen_cp_resume(rdev);
5262 r = r600_dma_resume(rdev);
5266 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5267 if (ring->ring_size) {
5268 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5271 r = uvd_v1_0_init(rdev);
5274 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5277 r = radeon_ib_pool_init(rdev);
5279 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5283 r = r600_audio_init(rdev);
5285 DRM_ERROR("radeon: audio init failed\n");
5292 int evergreen_resume(struct radeon_device *rdev)
5296 /* reset the asic, the gfx blocks are often in a bad state
5297 * after the driver is unloaded or after a resume
5299 if (radeon_asic_reset(rdev))
5300 dev_warn(rdev->dev, "GPU reset failed !\n");
5301 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5302 * posting will perform necessary task to bring back GPU into good
5306 atom_asic_init(rdev->mode_info.atom_context);
5308 /* init golden registers */
5309 evergreen_init_golden_registers(rdev);
5311 if (rdev->pm.pm_method == PM_METHOD_DPM)
5312 radeon_pm_resume(rdev);
5314 rdev->accel_working = true;
5315 r = evergreen_startup(rdev);
5317 DRM_ERROR("evergreen startup failed on resume\n");
5318 rdev->accel_working = false;
5326 int evergreen_suspend(struct radeon_device *rdev)
5328 radeon_pm_suspend(rdev);
5329 r600_audio_fini(rdev);
5330 uvd_v1_0_fini(rdev);
5331 radeon_uvd_suspend(rdev);
5333 r600_dma_stop(rdev);
5334 evergreen_irq_suspend(rdev);
5335 radeon_wb_disable(rdev);
5336 evergreen_pcie_gart_disable(rdev);
5341 /* Plan is to move initialization in that function and use
5342 * helper function so that radeon_device_init pretty much
5343 * do nothing more than calling asic specific function. This
5344 * should also allow to remove a bunch of callback function
5347 int evergreen_init(struct radeon_device *rdev)
5352 if (!radeon_get_bios(rdev)) {
5353 if (ASIC_IS_AVIVO(rdev))
5356 /* Must be an ATOMBIOS */
5357 if (!rdev->is_atom_bios) {
5358 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5361 r = radeon_atombios_init(rdev);
5364 /* reset the asic, the gfx blocks are often in a bad state
5365 * after the driver is unloaded or after a resume
5367 if (radeon_asic_reset(rdev))
5368 dev_warn(rdev->dev, "GPU reset failed !\n");
5369 /* Post card if necessary */
5370 if (!radeon_card_posted(rdev)) {
5372 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5375 DRM_INFO("GPU not posted. posting now...\n");
5376 atom_asic_init(rdev->mode_info.atom_context);
5378 /* init golden registers */
5379 evergreen_init_golden_registers(rdev);
5380 /* Initialize scratch registers */
5381 r600_scratch_init(rdev);
5382 /* Initialize surface registers */
5383 radeon_surface_init(rdev);
5384 /* Initialize clocks */
5385 radeon_get_clock_info(rdev->ddev);
5387 r = radeon_fence_driver_init(rdev);
5390 /* initialize AGP */
5391 if (rdev->flags & RADEON_IS_AGP) {
5392 r = radeon_agp_init(rdev);
5394 radeon_agp_disable(rdev);
5396 /* initialize memory controller */
5397 r = evergreen_mc_init(rdev);
5400 /* Memory manager */
5401 r = radeon_bo_init(rdev);
5405 if (ASIC_IS_DCE5(rdev)) {
5406 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5407 r = ni_init_microcode(rdev);
5409 DRM_ERROR("Failed to load firmware!\n");
5414 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5415 r = r600_init_microcode(rdev);
5417 DRM_ERROR("Failed to load firmware!\n");
5423 /* Initialize power management */
5424 radeon_pm_init(rdev);
5426 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5427 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5429 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5430 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5432 r = radeon_uvd_init(rdev);
5434 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5435 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5439 rdev->ih.ring_obj = NULL;
5440 r600_ih_ring_init(rdev, 64 * 1024);
5442 r = r600_pcie_gart_init(rdev);
5446 rdev->accel_working = true;
5447 r = evergreen_startup(rdev);
5449 dev_err(rdev->dev, "disabling GPU acceleration\n");
5451 r600_dma_fini(rdev);
5452 r600_irq_fini(rdev);
5453 if (rdev->flags & RADEON_IS_IGP)
5454 sumo_rlc_fini(rdev);
5455 radeon_wb_fini(rdev);
5456 radeon_ib_pool_fini(rdev);
5457 radeon_irq_kms_fini(rdev);
5458 evergreen_pcie_gart_fini(rdev);
5459 rdev->accel_working = false;
5462 /* Don't start up if the MC ucode is missing on BTC parts.
5463 * The default clocks and voltages before the MC ucode
5464 * is loaded are not suffient for advanced operations.
5466 if (ASIC_IS_DCE5(rdev)) {
5467 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5468 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5476 void evergreen_fini(struct radeon_device *rdev)
5478 radeon_pm_fini(rdev);
5479 r600_audio_fini(rdev);
5481 r600_dma_fini(rdev);
5482 r600_irq_fini(rdev);
5483 if (rdev->flags & RADEON_IS_IGP)
5484 sumo_rlc_fini(rdev);
5485 radeon_wb_fini(rdev);
5486 radeon_ib_pool_fini(rdev);
5487 radeon_irq_kms_fini(rdev);
5488 uvd_v1_0_fini(rdev);
5489 radeon_uvd_fini(rdev);
5490 evergreen_pcie_gart_fini(rdev);
5491 r600_vram_scratch_fini(rdev);
5492 radeon_gem_fini(rdev);
5493 radeon_fence_driver_fini(rdev);
5494 radeon_agp_fini(rdev);
5495 radeon_bo_fini(rdev);
5496 radeon_atombios_fini(rdev);
5501 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5503 u32 link_width_cntl, speed_cntl;
5505 if (radeon_pcie_gen2 == 0)
5508 if (rdev->flags & RADEON_IS_IGP)
5511 if (!(rdev->flags & RADEON_IS_PCIE))
5514 /* x2 cards have a special sequence */
5515 if (ASIC_IS_X2(rdev))
5518 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5519 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5522 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5523 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5524 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5528 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5530 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5531 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5533 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5534 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5535 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5537 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5538 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5539 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5541 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5542 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5543 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5545 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5546 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5547 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5549 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5550 speed_cntl |= LC_GEN2_EN_STRAP;
5551 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5554 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5555 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5557 link_width_cntl |= LC_UPCONFIGURE_DIS;
5559 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5560 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5564 void evergreen_program_aspm(struct radeon_device *rdev)
5567 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5568 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5569 /* fusion_platform = true
5570 * if the system is a fusion system
5571 * (APU or DGPU in a fusion system).
5572 * todo: check if the system is a fusion platform.
5574 bool fusion_platform = false;
5576 if (radeon_aspm == 0)
5579 if (!(rdev->flags & RADEON_IS_PCIE))
5582 switch (rdev->family) {
5595 disable_l0s = false;
5599 if (rdev->flags & RADEON_IS_IGP)
5600 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5602 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5603 if (fusion_platform)
5608 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5610 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5611 if (fusion_platform)
5616 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5618 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5619 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5621 if (rdev->family >= CHIP_BARTS)
5622 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5624 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5628 if (rdev->family >= CHIP_BARTS)
5629 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5631 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5633 if (!disable_plloff_in_l1) {
5634 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5635 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5636 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5638 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5640 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5641 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5642 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5644 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5646 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5647 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5648 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5650 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5652 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5653 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5654 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5656 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5658 if (rdev->family >= CHIP_BARTS) {
5659 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5660 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5661 data |= PLL_RAMP_UP_TIME_0(4);
5663 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5665 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5666 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5667 data |= PLL_RAMP_UP_TIME_1(4);
5669 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5671 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5672 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5673 data |= PLL_RAMP_UP_TIME_0(4);
5675 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5677 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5678 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5679 data |= PLL_RAMP_UP_TIME_1(4);
5681 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5684 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5685 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5686 data |= LC_DYN_LANES_PWR_STATE(3);
5688 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5690 if (rdev->family >= CHIP_BARTS) {
5691 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5692 data &= ~LS2_EXIT_TIME_MASK;
5693 data |= LS2_EXIT_TIME(1);
5695 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5697 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5698 data &= ~LS2_EXIT_TIME_MASK;
5699 data |= LS2_EXIT_TIME(1);
5701 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5706 /* evergreen parts only */
5707 if (rdev->family < CHIP_BARTS)
5708 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5710 if (pcie_lc_cntl != pcie_lc_cntl_old)
5711 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);