2 * Copyright 2010 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
36 #include "radeon_ucode.h"
38 static const u32 crtc_offsets[6] =
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
48 #include "clearstate_evergreen.h"
50 static u32 sumo_rlc_save_restore_register_list[] =
134 static u32 sumo_rlc_save_restore_register_list_size = ARRAY_SIZE(sumo_rlc_save_restore_register_list);
136 static void evergreen_gpu_init(struct radeon_device *rdev);
137 void evergreen_fini(struct radeon_device *rdev);
138 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
139 void evergreen_program_aspm(struct radeon_device *rdev);
140 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
141 int ring, u32 cp_int_cntl);
142 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
143 u32 status, u32 addr);
145 static const u32 evergreen_golden_registers[] =
147 0x3f90, 0xffff0000, 0xff000000,
148 0x9148, 0xffff0000, 0xff000000,
149 0x3f94, 0xffff0000, 0xff000000,
150 0x914c, 0xffff0000, 0xff000000,
151 0x9b7c, 0xffffffff, 0x00000000,
152 0x8a14, 0xffffffff, 0x00000007,
153 0x8b10, 0xffffffff, 0x00000000,
154 0x960c, 0xffffffff, 0x54763210,
155 0x88c4, 0xffffffff, 0x000000c2,
156 0x88d4, 0xffffffff, 0x00000010,
157 0x8974, 0xffffffff, 0x00000000,
158 0xc78, 0x00000080, 0x00000080,
159 0x5eb4, 0xffffffff, 0x00000002,
160 0x5e78, 0xffffffff, 0x001000f0,
161 0x6104, 0x01000300, 0x00000000,
162 0x5bc0, 0x00300000, 0x00000000,
163 0x7030, 0xffffffff, 0x00000011,
164 0x7c30, 0xffffffff, 0x00000011,
165 0x10830, 0xffffffff, 0x00000011,
166 0x11430, 0xffffffff, 0x00000011,
167 0x12030, 0xffffffff, 0x00000011,
168 0x12c30, 0xffffffff, 0x00000011,
169 0xd02c, 0xffffffff, 0x08421000,
170 0x240c, 0xffffffff, 0x00000380,
171 0x8b24, 0xffffffff, 0x00ff0fff,
172 0x28a4c, 0x06000000, 0x06000000,
173 0x10c, 0x00000001, 0x00000001,
174 0x8d00, 0xffffffff, 0x100e4848,
175 0x8d04, 0xffffffff, 0x00164745,
176 0x8c00, 0xffffffff, 0xe4000003,
177 0x8c04, 0xffffffff, 0x40600060,
178 0x8c08, 0xffffffff, 0x001c001c,
179 0x8cf0, 0xffffffff, 0x08e00620,
180 0x8c20, 0xffffffff, 0x00800080,
181 0x8c24, 0xffffffff, 0x00800080,
182 0x8c18, 0xffffffff, 0x20202078,
183 0x8c1c, 0xffffffff, 0x00001010,
184 0x28350, 0xffffffff, 0x00000000,
185 0xa008, 0xffffffff, 0x00010000,
186 0x5cc, 0xffffffff, 0x00000001,
187 0x9508, 0xffffffff, 0x00000002,
188 0x913c, 0x0000000f, 0x0000000a
191 static const u32 evergreen_golden_registers2[] =
193 0x2f4c, 0xffffffff, 0x00000000,
194 0x54f4, 0xffffffff, 0x00000000,
195 0x54f0, 0xffffffff, 0x00000000,
196 0x5498, 0xffffffff, 0x00000000,
197 0x549c, 0xffffffff, 0x00000000,
198 0x5494, 0xffffffff, 0x00000000,
199 0x53cc, 0xffffffff, 0x00000000,
200 0x53c8, 0xffffffff, 0x00000000,
201 0x53c4, 0xffffffff, 0x00000000,
202 0x53c0, 0xffffffff, 0x00000000,
203 0x53bc, 0xffffffff, 0x00000000,
204 0x53b8, 0xffffffff, 0x00000000,
205 0x53b4, 0xffffffff, 0x00000000,
206 0x53b0, 0xffffffff, 0x00000000
209 static const u32 cypress_mgcg_init[] =
211 0x802c, 0xffffffff, 0xc0000000,
212 0x5448, 0xffffffff, 0x00000100,
213 0x55e4, 0xffffffff, 0x00000100,
214 0x160c, 0xffffffff, 0x00000100,
215 0x5644, 0xffffffff, 0x00000100,
216 0xc164, 0xffffffff, 0x00000100,
217 0x8a18, 0xffffffff, 0x00000100,
218 0x897c, 0xffffffff, 0x06000100,
219 0x8b28, 0xffffffff, 0x00000100,
220 0x9144, 0xffffffff, 0x00000100,
221 0x9a60, 0xffffffff, 0x00000100,
222 0x9868, 0xffffffff, 0x00000100,
223 0x8d58, 0xffffffff, 0x00000100,
224 0x9510, 0xffffffff, 0x00000100,
225 0x949c, 0xffffffff, 0x00000100,
226 0x9654, 0xffffffff, 0x00000100,
227 0x9030, 0xffffffff, 0x00000100,
228 0x9034, 0xffffffff, 0x00000100,
229 0x9038, 0xffffffff, 0x00000100,
230 0x903c, 0xffffffff, 0x00000100,
231 0x9040, 0xffffffff, 0x00000100,
232 0xa200, 0xffffffff, 0x00000100,
233 0xa204, 0xffffffff, 0x00000100,
234 0xa208, 0xffffffff, 0x00000100,
235 0xa20c, 0xffffffff, 0x00000100,
236 0x971c, 0xffffffff, 0x00000100,
237 0x977c, 0xffffffff, 0x00000100,
238 0x3f80, 0xffffffff, 0x00000100,
239 0xa210, 0xffffffff, 0x00000100,
240 0xa214, 0xffffffff, 0x00000100,
241 0x4d8, 0xffffffff, 0x00000100,
242 0x9784, 0xffffffff, 0x00000100,
243 0x9698, 0xffffffff, 0x00000100,
244 0x4d4, 0xffffffff, 0x00000200,
245 0x30cc, 0xffffffff, 0x00000100,
246 0xd0c0, 0xffffffff, 0xff000100,
247 0x802c, 0xffffffff, 0x40000000,
248 0x915c, 0xffffffff, 0x00010000,
249 0x9160, 0xffffffff, 0x00030002,
250 0x9178, 0xffffffff, 0x00070000,
251 0x917c, 0xffffffff, 0x00030002,
252 0x9180, 0xffffffff, 0x00050004,
253 0x918c, 0xffffffff, 0x00010006,
254 0x9190, 0xffffffff, 0x00090008,
255 0x9194, 0xffffffff, 0x00070000,
256 0x9198, 0xffffffff, 0x00030002,
257 0x919c, 0xffffffff, 0x00050004,
258 0x91a8, 0xffffffff, 0x00010006,
259 0x91ac, 0xffffffff, 0x00090008,
260 0x91b0, 0xffffffff, 0x00070000,
261 0x91b4, 0xffffffff, 0x00030002,
262 0x91b8, 0xffffffff, 0x00050004,
263 0x91c4, 0xffffffff, 0x00010006,
264 0x91c8, 0xffffffff, 0x00090008,
265 0x91cc, 0xffffffff, 0x00070000,
266 0x91d0, 0xffffffff, 0x00030002,
267 0x91d4, 0xffffffff, 0x00050004,
268 0x91e0, 0xffffffff, 0x00010006,
269 0x91e4, 0xffffffff, 0x00090008,
270 0x91e8, 0xffffffff, 0x00000000,
271 0x91ec, 0xffffffff, 0x00070000,
272 0x91f0, 0xffffffff, 0x00030002,
273 0x91f4, 0xffffffff, 0x00050004,
274 0x9200, 0xffffffff, 0x00010006,
275 0x9204, 0xffffffff, 0x00090008,
276 0x9208, 0xffffffff, 0x00070000,
277 0x920c, 0xffffffff, 0x00030002,
278 0x9210, 0xffffffff, 0x00050004,
279 0x921c, 0xffffffff, 0x00010006,
280 0x9220, 0xffffffff, 0x00090008,
281 0x9224, 0xffffffff, 0x00070000,
282 0x9228, 0xffffffff, 0x00030002,
283 0x922c, 0xffffffff, 0x00050004,
284 0x9238, 0xffffffff, 0x00010006,
285 0x923c, 0xffffffff, 0x00090008,
286 0x9240, 0xffffffff, 0x00070000,
287 0x9244, 0xffffffff, 0x00030002,
288 0x9248, 0xffffffff, 0x00050004,
289 0x9254, 0xffffffff, 0x00010006,
290 0x9258, 0xffffffff, 0x00090008,
291 0x925c, 0xffffffff, 0x00070000,
292 0x9260, 0xffffffff, 0x00030002,
293 0x9264, 0xffffffff, 0x00050004,
294 0x9270, 0xffffffff, 0x00010006,
295 0x9274, 0xffffffff, 0x00090008,
296 0x9278, 0xffffffff, 0x00070000,
297 0x927c, 0xffffffff, 0x00030002,
298 0x9280, 0xffffffff, 0x00050004,
299 0x928c, 0xffffffff, 0x00010006,
300 0x9290, 0xffffffff, 0x00090008,
301 0x9294, 0xffffffff, 0x00000000,
302 0x929c, 0xffffffff, 0x00000001,
303 0x802c, 0xffffffff, 0x40010000,
304 0x915c, 0xffffffff, 0x00010000,
305 0x9160, 0xffffffff, 0x00030002,
306 0x9178, 0xffffffff, 0x00070000,
307 0x917c, 0xffffffff, 0x00030002,
308 0x9180, 0xffffffff, 0x00050004,
309 0x918c, 0xffffffff, 0x00010006,
310 0x9190, 0xffffffff, 0x00090008,
311 0x9194, 0xffffffff, 0x00070000,
312 0x9198, 0xffffffff, 0x00030002,
313 0x919c, 0xffffffff, 0x00050004,
314 0x91a8, 0xffffffff, 0x00010006,
315 0x91ac, 0xffffffff, 0x00090008,
316 0x91b0, 0xffffffff, 0x00070000,
317 0x91b4, 0xffffffff, 0x00030002,
318 0x91b8, 0xffffffff, 0x00050004,
319 0x91c4, 0xffffffff, 0x00010006,
320 0x91c8, 0xffffffff, 0x00090008,
321 0x91cc, 0xffffffff, 0x00070000,
322 0x91d0, 0xffffffff, 0x00030002,
323 0x91d4, 0xffffffff, 0x00050004,
324 0x91e0, 0xffffffff, 0x00010006,
325 0x91e4, 0xffffffff, 0x00090008,
326 0x91e8, 0xffffffff, 0x00000000,
327 0x91ec, 0xffffffff, 0x00070000,
328 0x91f0, 0xffffffff, 0x00030002,
329 0x91f4, 0xffffffff, 0x00050004,
330 0x9200, 0xffffffff, 0x00010006,
331 0x9204, 0xffffffff, 0x00090008,
332 0x9208, 0xffffffff, 0x00070000,
333 0x920c, 0xffffffff, 0x00030002,
334 0x9210, 0xffffffff, 0x00050004,
335 0x921c, 0xffffffff, 0x00010006,
336 0x9220, 0xffffffff, 0x00090008,
337 0x9224, 0xffffffff, 0x00070000,
338 0x9228, 0xffffffff, 0x00030002,
339 0x922c, 0xffffffff, 0x00050004,
340 0x9238, 0xffffffff, 0x00010006,
341 0x923c, 0xffffffff, 0x00090008,
342 0x9240, 0xffffffff, 0x00070000,
343 0x9244, 0xffffffff, 0x00030002,
344 0x9248, 0xffffffff, 0x00050004,
345 0x9254, 0xffffffff, 0x00010006,
346 0x9258, 0xffffffff, 0x00090008,
347 0x925c, 0xffffffff, 0x00070000,
348 0x9260, 0xffffffff, 0x00030002,
349 0x9264, 0xffffffff, 0x00050004,
350 0x9270, 0xffffffff, 0x00010006,
351 0x9274, 0xffffffff, 0x00090008,
352 0x9278, 0xffffffff, 0x00070000,
353 0x927c, 0xffffffff, 0x00030002,
354 0x9280, 0xffffffff, 0x00050004,
355 0x928c, 0xffffffff, 0x00010006,
356 0x9290, 0xffffffff, 0x00090008,
357 0x9294, 0xffffffff, 0x00000000,
358 0x929c, 0xffffffff, 0x00000001,
359 0x802c, 0xffffffff, 0xc0000000
362 static const u32 redwood_mgcg_init[] =
364 0x802c, 0xffffffff, 0xc0000000,
365 0x5448, 0xffffffff, 0x00000100,
366 0x55e4, 0xffffffff, 0x00000100,
367 0x160c, 0xffffffff, 0x00000100,
368 0x5644, 0xffffffff, 0x00000100,
369 0xc164, 0xffffffff, 0x00000100,
370 0x8a18, 0xffffffff, 0x00000100,
371 0x897c, 0xffffffff, 0x06000100,
372 0x8b28, 0xffffffff, 0x00000100,
373 0x9144, 0xffffffff, 0x00000100,
374 0x9a60, 0xffffffff, 0x00000100,
375 0x9868, 0xffffffff, 0x00000100,
376 0x8d58, 0xffffffff, 0x00000100,
377 0x9510, 0xffffffff, 0x00000100,
378 0x949c, 0xffffffff, 0x00000100,
379 0x9654, 0xffffffff, 0x00000100,
380 0x9030, 0xffffffff, 0x00000100,
381 0x9034, 0xffffffff, 0x00000100,
382 0x9038, 0xffffffff, 0x00000100,
383 0x903c, 0xffffffff, 0x00000100,
384 0x9040, 0xffffffff, 0x00000100,
385 0xa200, 0xffffffff, 0x00000100,
386 0xa204, 0xffffffff, 0x00000100,
387 0xa208, 0xffffffff, 0x00000100,
388 0xa20c, 0xffffffff, 0x00000100,
389 0x971c, 0xffffffff, 0x00000100,
390 0x977c, 0xffffffff, 0x00000100,
391 0x3f80, 0xffffffff, 0x00000100,
392 0xa210, 0xffffffff, 0x00000100,
393 0xa214, 0xffffffff, 0x00000100,
394 0x4d8, 0xffffffff, 0x00000100,
395 0x9784, 0xffffffff, 0x00000100,
396 0x9698, 0xffffffff, 0x00000100,
397 0x4d4, 0xffffffff, 0x00000200,
398 0x30cc, 0xffffffff, 0x00000100,
399 0xd0c0, 0xffffffff, 0xff000100,
400 0x802c, 0xffffffff, 0x40000000,
401 0x915c, 0xffffffff, 0x00010000,
402 0x9160, 0xffffffff, 0x00030002,
403 0x9178, 0xffffffff, 0x00070000,
404 0x917c, 0xffffffff, 0x00030002,
405 0x9180, 0xffffffff, 0x00050004,
406 0x918c, 0xffffffff, 0x00010006,
407 0x9190, 0xffffffff, 0x00090008,
408 0x9194, 0xffffffff, 0x00070000,
409 0x9198, 0xffffffff, 0x00030002,
410 0x919c, 0xffffffff, 0x00050004,
411 0x91a8, 0xffffffff, 0x00010006,
412 0x91ac, 0xffffffff, 0x00090008,
413 0x91b0, 0xffffffff, 0x00070000,
414 0x91b4, 0xffffffff, 0x00030002,
415 0x91b8, 0xffffffff, 0x00050004,
416 0x91c4, 0xffffffff, 0x00010006,
417 0x91c8, 0xffffffff, 0x00090008,
418 0x91cc, 0xffffffff, 0x00070000,
419 0x91d0, 0xffffffff, 0x00030002,
420 0x91d4, 0xffffffff, 0x00050004,
421 0x91e0, 0xffffffff, 0x00010006,
422 0x91e4, 0xffffffff, 0x00090008,
423 0x91e8, 0xffffffff, 0x00000000,
424 0x91ec, 0xffffffff, 0x00070000,
425 0x91f0, 0xffffffff, 0x00030002,
426 0x91f4, 0xffffffff, 0x00050004,
427 0x9200, 0xffffffff, 0x00010006,
428 0x9204, 0xffffffff, 0x00090008,
429 0x9294, 0xffffffff, 0x00000000,
430 0x929c, 0xffffffff, 0x00000001,
431 0x802c, 0xffffffff, 0xc0000000
434 static const u32 cedar_golden_registers[] =
436 0x3f90, 0xffff0000, 0xff000000,
437 0x9148, 0xffff0000, 0xff000000,
438 0x3f94, 0xffff0000, 0xff000000,
439 0x914c, 0xffff0000, 0xff000000,
440 0x9b7c, 0xffffffff, 0x00000000,
441 0x8a14, 0xffffffff, 0x00000007,
442 0x8b10, 0xffffffff, 0x00000000,
443 0x960c, 0xffffffff, 0x54763210,
444 0x88c4, 0xffffffff, 0x000000c2,
445 0x88d4, 0xffffffff, 0x00000000,
446 0x8974, 0xffffffff, 0x00000000,
447 0xc78, 0x00000080, 0x00000080,
448 0x5eb4, 0xffffffff, 0x00000002,
449 0x5e78, 0xffffffff, 0x001000f0,
450 0x6104, 0x01000300, 0x00000000,
451 0x5bc0, 0x00300000, 0x00000000,
452 0x7030, 0xffffffff, 0x00000011,
453 0x7c30, 0xffffffff, 0x00000011,
454 0x10830, 0xffffffff, 0x00000011,
455 0x11430, 0xffffffff, 0x00000011,
456 0xd02c, 0xffffffff, 0x08421000,
457 0x240c, 0xffffffff, 0x00000380,
458 0x8b24, 0xffffffff, 0x00ff0fff,
459 0x28a4c, 0x06000000, 0x06000000,
460 0x10c, 0x00000001, 0x00000001,
461 0x8d00, 0xffffffff, 0x100e4848,
462 0x8d04, 0xffffffff, 0x00164745,
463 0x8c00, 0xffffffff, 0xe4000003,
464 0x8c04, 0xffffffff, 0x40600060,
465 0x8c08, 0xffffffff, 0x001c001c,
466 0x8cf0, 0xffffffff, 0x08e00410,
467 0x8c20, 0xffffffff, 0x00800080,
468 0x8c24, 0xffffffff, 0x00800080,
469 0x8c18, 0xffffffff, 0x20202078,
470 0x8c1c, 0xffffffff, 0x00001010,
471 0x28350, 0xffffffff, 0x00000000,
472 0xa008, 0xffffffff, 0x00010000,
473 0x5cc, 0xffffffff, 0x00000001,
474 0x9508, 0xffffffff, 0x00000002
477 static const u32 cedar_mgcg_init[] =
479 0x802c, 0xffffffff, 0xc0000000,
480 0x5448, 0xffffffff, 0x00000100,
481 0x55e4, 0xffffffff, 0x00000100,
482 0x160c, 0xffffffff, 0x00000100,
483 0x5644, 0xffffffff, 0x00000100,
484 0xc164, 0xffffffff, 0x00000100,
485 0x8a18, 0xffffffff, 0x00000100,
486 0x897c, 0xffffffff, 0x06000100,
487 0x8b28, 0xffffffff, 0x00000100,
488 0x9144, 0xffffffff, 0x00000100,
489 0x9a60, 0xffffffff, 0x00000100,
490 0x9868, 0xffffffff, 0x00000100,
491 0x8d58, 0xffffffff, 0x00000100,
492 0x9510, 0xffffffff, 0x00000100,
493 0x949c, 0xffffffff, 0x00000100,
494 0x9654, 0xffffffff, 0x00000100,
495 0x9030, 0xffffffff, 0x00000100,
496 0x9034, 0xffffffff, 0x00000100,
497 0x9038, 0xffffffff, 0x00000100,
498 0x903c, 0xffffffff, 0x00000100,
499 0x9040, 0xffffffff, 0x00000100,
500 0xa200, 0xffffffff, 0x00000100,
501 0xa204, 0xffffffff, 0x00000100,
502 0xa208, 0xffffffff, 0x00000100,
503 0xa20c, 0xffffffff, 0x00000100,
504 0x971c, 0xffffffff, 0x00000100,
505 0x977c, 0xffffffff, 0x00000100,
506 0x3f80, 0xffffffff, 0x00000100,
507 0xa210, 0xffffffff, 0x00000100,
508 0xa214, 0xffffffff, 0x00000100,
509 0x4d8, 0xffffffff, 0x00000100,
510 0x9784, 0xffffffff, 0x00000100,
511 0x9698, 0xffffffff, 0x00000100,
512 0x4d4, 0xffffffff, 0x00000200,
513 0x30cc, 0xffffffff, 0x00000100,
514 0xd0c0, 0xffffffff, 0xff000100,
515 0x802c, 0xffffffff, 0x40000000,
516 0x915c, 0xffffffff, 0x00010000,
517 0x9178, 0xffffffff, 0x00050000,
518 0x917c, 0xffffffff, 0x00030002,
519 0x918c, 0xffffffff, 0x00010004,
520 0x9190, 0xffffffff, 0x00070006,
521 0x9194, 0xffffffff, 0x00050000,
522 0x9198, 0xffffffff, 0x00030002,
523 0x91a8, 0xffffffff, 0x00010004,
524 0x91ac, 0xffffffff, 0x00070006,
525 0x91e8, 0xffffffff, 0x00000000,
526 0x9294, 0xffffffff, 0x00000000,
527 0x929c, 0xffffffff, 0x00000001,
528 0x802c, 0xffffffff, 0xc0000000
531 static const u32 juniper_mgcg_init[] =
533 0x802c, 0xffffffff, 0xc0000000,
534 0x5448, 0xffffffff, 0x00000100,
535 0x55e4, 0xffffffff, 0x00000100,
536 0x160c, 0xffffffff, 0x00000100,
537 0x5644, 0xffffffff, 0x00000100,
538 0xc164, 0xffffffff, 0x00000100,
539 0x8a18, 0xffffffff, 0x00000100,
540 0x897c, 0xffffffff, 0x06000100,
541 0x8b28, 0xffffffff, 0x00000100,
542 0x9144, 0xffffffff, 0x00000100,
543 0x9a60, 0xffffffff, 0x00000100,
544 0x9868, 0xffffffff, 0x00000100,
545 0x8d58, 0xffffffff, 0x00000100,
546 0x9510, 0xffffffff, 0x00000100,
547 0x949c, 0xffffffff, 0x00000100,
548 0x9654, 0xffffffff, 0x00000100,
549 0x9030, 0xffffffff, 0x00000100,
550 0x9034, 0xffffffff, 0x00000100,
551 0x9038, 0xffffffff, 0x00000100,
552 0x903c, 0xffffffff, 0x00000100,
553 0x9040, 0xffffffff, 0x00000100,
554 0xa200, 0xffffffff, 0x00000100,
555 0xa204, 0xffffffff, 0x00000100,
556 0xa208, 0xffffffff, 0x00000100,
557 0xa20c, 0xffffffff, 0x00000100,
558 0x971c, 0xffffffff, 0x00000100,
559 0xd0c0, 0xffffffff, 0xff000100,
560 0x802c, 0xffffffff, 0x40000000,
561 0x915c, 0xffffffff, 0x00010000,
562 0x9160, 0xffffffff, 0x00030002,
563 0x9178, 0xffffffff, 0x00070000,
564 0x917c, 0xffffffff, 0x00030002,
565 0x9180, 0xffffffff, 0x00050004,
566 0x918c, 0xffffffff, 0x00010006,
567 0x9190, 0xffffffff, 0x00090008,
568 0x9194, 0xffffffff, 0x00070000,
569 0x9198, 0xffffffff, 0x00030002,
570 0x919c, 0xffffffff, 0x00050004,
571 0x91a8, 0xffffffff, 0x00010006,
572 0x91ac, 0xffffffff, 0x00090008,
573 0x91b0, 0xffffffff, 0x00070000,
574 0x91b4, 0xffffffff, 0x00030002,
575 0x91b8, 0xffffffff, 0x00050004,
576 0x91c4, 0xffffffff, 0x00010006,
577 0x91c8, 0xffffffff, 0x00090008,
578 0x91cc, 0xffffffff, 0x00070000,
579 0x91d0, 0xffffffff, 0x00030002,
580 0x91d4, 0xffffffff, 0x00050004,
581 0x91e0, 0xffffffff, 0x00010006,
582 0x91e4, 0xffffffff, 0x00090008,
583 0x91e8, 0xffffffff, 0x00000000,
584 0x91ec, 0xffffffff, 0x00070000,
585 0x91f0, 0xffffffff, 0x00030002,
586 0x91f4, 0xffffffff, 0x00050004,
587 0x9200, 0xffffffff, 0x00010006,
588 0x9204, 0xffffffff, 0x00090008,
589 0x9208, 0xffffffff, 0x00070000,
590 0x920c, 0xffffffff, 0x00030002,
591 0x9210, 0xffffffff, 0x00050004,
592 0x921c, 0xffffffff, 0x00010006,
593 0x9220, 0xffffffff, 0x00090008,
594 0x9224, 0xffffffff, 0x00070000,
595 0x9228, 0xffffffff, 0x00030002,
596 0x922c, 0xffffffff, 0x00050004,
597 0x9238, 0xffffffff, 0x00010006,
598 0x923c, 0xffffffff, 0x00090008,
599 0x9240, 0xffffffff, 0x00070000,
600 0x9244, 0xffffffff, 0x00030002,
601 0x9248, 0xffffffff, 0x00050004,
602 0x9254, 0xffffffff, 0x00010006,
603 0x9258, 0xffffffff, 0x00090008,
604 0x925c, 0xffffffff, 0x00070000,
605 0x9260, 0xffffffff, 0x00030002,
606 0x9264, 0xffffffff, 0x00050004,
607 0x9270, 0xffffffff, 0x00010006,
608 0x9274, 0xffffffff, 0x00090008,
609 0x9278, 0xffffffff, 0x00070000,
610 0x927c, 0xffffffff, 0x00030002,
611 0x9280, 0xffffffff, 0x00050004,
612 0x928c, 0xffffffff, 0x00010006,
613 0x9290, 0xffffffff, 0x00090008,
614 0x9294, 0xffffffff, 0x00000000,
615 0x929c, 0xffffffff, 0x00000001,
616 0x802c, 0xffffffff, 0xc0000000,
617 0x977c, 0xffffffff, 0x00000100,
618 0x3f80, 0xffffffff, 0x00000100,
619 0xa210, 0xffffffff, 0x00000100,
620 0xa214, 0xffffffff, 0x00000100,
621 0x4d8, 0xffffffff, 0x00000100,
622 0x9784, 0xffffffff, 0x00000100,
623 0x9698, 0xffffffff, 0x00000100,
624 0x4d4, 0xffffffff, 0x00000200,
625 0x30cc, 0xffffffff, 0x00000100,
626 0x802c, 0xffffffff, 0xc0000000
629 static const u32 supersumo_golden_registers[] =
631 0x5eb4, 0xffffffff, 0x00000002,
632 0x5cc, 0xffffffff, 0x00000001,
633 0x7030, 0xffffffff, 0x00000011,
634 0x7c30, 0xffffffff, 0x00000011,
635 0x6104, 0x01000300, 0x00000000,
636 0x5bc0, 0x00300000, 0x00000000,
637 0x8c04, 0xffffffff, 0x40600060,
638 0x8c08, 0xffffffff, 0x001c001c,
639 0x8c20, 0xffffffff, 0x00800080,
640 0x8c24, 0xffffffff, 0x00800080,
641 0x8c18, 0xffffffff, 0x20202078,
642 0x8c1c, 0xffffffff, 0x00001010,
643 0x918c, 0xffffffff, 0x00010006,
644 0x91a8, 0xffffffff, 0x00010006,
645 0x91c4, 0xffffffff, 0x00010006,
646 0x91e0, 0xffffffff, 0x00010006,
647 0x9200, 0xffffffff, 0x00010006,
648 0x9150, 0xffffffff, 0x6e944040,
649 0x917c, 0xffffffff, 0x00030002,
650 0x9180, 0xffffffff, 0x00050004,
651 0x9198, 0xffffffff, 0x00030002,
652 0x919c, 0xffffffff, 0x00050004,
653 0x91b4, 0xffffffff, 0x00030002,
654 0x91b8, 0xffffffff, 0x00050004,
655 0x91d0, 0xffffffff, 0x00030002,
656 0x91d4, 0xffffffff, 0x00050004,
657 0x91f0, 0xffffffff, 0x00030002,
658 0x91f4, 0xffffffff, 0x00050004,
659 0x915c, 0xffffffff, 0x00010000,
660 0x9160, 0xffffffff, 0x00030002,
661 0x3f90, 0xffff0000, 0xff000000,
662 0x9178, 0xffffffff, 0x00070000,
663 0x9194, 0xffffffff, 0x00070000,
664 0x91b0, 0xffffffff, 0x00070000,
665 0x91cc, 0xffffffff, 0x00070000,
666 0x91ec, 0xffffffff, 0x00070000,
667 0x9148, 0xffff0000, 0xff000000,
668 0x9190, 0xffffffff, 0x00090008,
669 0x91ac, 0xffffffff, 0x00090008,
670 0x91c8, 0xffffffff, 0x00090008,
671 0x91e4, 0xffffffff, 0x00090008,
672 0x9204, 0xffffffff, 0x00090008,
673 0x3f94, 0xffff0000, 0xff000000,
674 0x914c, 0xffff0000, 0xff000000,
675 0x929c, 0xffffffff, 0x00000001,
676 0x8a18, 0xffffffff, 0x00000100,
677 0x8b28, 0xffffffff, 0x00000100,
678 0x9144, 0xffffffff, 0x00000100,
679 0x5644, 0xffffffff, 0x00000100,
680 0x9b7c, 0xffffffff, 0x00000000,
681 0x8030, 0xffffffff, 0x0000100a,
682 0x8a14, 0xffffffff, 0x00000007,
683 0x8b24, 0xffffffff, 0x00ff0fff,
684 0x8b10, 0xffffffff, 0x00000000,
685 0x28a4c, 0x06000000, 0x06000000,
686 0x4d8, 0xffffffff, 0x00000100,
687 0x913c, 0xffff000f, 0x0100000a,
688 0x960c, 0xffffffff, 0x54763210,
689 0x88c4, 0xffffffff, 0x000000c2,
690 0x88d4, 0xffffffff, 0x00000010,
691 0x8974, 0xffffffff, 0x00000000,
692 0xc78, 0x00000080, 0x00000080,
693 0x5e78, 0xffffffff, 0x001000f0,
694 0xd02c, 0xffffffff, 0x08421000,
695 0xa008, 0xffffffff, 0x00010000,
696 0x8d00, 0xffffffff, 0x100e4848,
697 0x8d04, 0xffffffff, 0x00164745,
698 0x8c00, 0xffffffff, 0xe4000003,
699 0x8cf0, 0x1fffffff, 0x08e00620,
700 0x28350, 0xffffffff, 0x00000000,
701 0x9508, 0xffffffff, 0x00000002
704 static const u32 sumo_golden_registers[] =
706 0x900c, 0x00ffffff, 0x0017071f,
707 0x8c18, 0xffffffff, 0x10101060,
708 0x8c1c, 0xffffffff, 0x00001010,
709 0x8c30, 0x0000000f, 0x00000005,
710 0x9688, 0x0000000f, 0x00000007
713 static const u32 wrestler_golden_registers[] =
715 0x5eb4, 0xffffffff, 0x00000002,
716 0x5cc, 0xffffffff, 0x00000001,
717 0x7030, 0xffffffff, 0x00000011,
718 0x7c30, 0xffffffff, 0x00000011,
719 0x6104, 0x01000300, 0x00000000,
720 0x5bc0, 0x00300000, 0x00000000,
721 0x918c, 0xffffffff, 0x00010006,
722 0x91a8, 0xffffffff, 0x00010006,
723 0x9150, 0xffffffff, 0x6e944040,
724 0x917c, 0xffffffff, 0x00030002,
725 0x9198, 0xffffffff, 0x00030002,
726 0x915c, 0xffffffff, 0x00010000,
727 0x3f90, 0xffff0000, 0xff000000,
728 0x9178, 0xffffffff, 0x00070000,
729 0x9194, 0xffffffff, 0x00070000,
730 0x9148, 0xffff0000, 0xff000000,
731 0x9190, 0xffffffff, 0x00090008,
732 0x91ac, 0xffffffff, 0x00090008,
733 0x3f94, 0xffff0000, 0xff000000,
734 0x914c, 0xffff0000, 0xff000000,
735 0x929c, 0xffffffff, 0x00000001,
736 0x8a18, 0xffffffff, 0x00000100,
737 0x8b28, 0xffffffff, 0x00000100,
738 0x9144, 0xffffffff, 0x00000100,
739 0x9b7c, 0xffffffff, 0x00000000,
740 0x8030, 0xffffffff, 0x0000100a,
741 0x8a14, 0xffffffff, 0x00000001,
742 0x8b24, 0xffffffff, 0x00ff0fff,
743 0x8b10, 0xffffffff, 0x00000000,
744 0x28a4c, 0x06000000, 0x06000000,
745 0x4d8, 0xffffffff, 0x00000100,
746 0x913c, 0xffff000f, 0x0100000a,
747 0x960c, 0xffffffff, 0x54763210,
748 0x88c4, 0xffffffff, 0x000000c2,
749 0x88d4, 0xffffffff, 0x00000010,
750 0x8974, 0xffffffff, 0x00000000,
751 0xc78, 0x00000080, 0x00000080,
752 0x5e78, 0xffffffff, 0x001000f0,
753 0xd02c, 0xffffffff, 0x08421000,
754 0xa008, 0xffffffff, 0x00010000,
755 0x8d00, 0xffffffff, 0x100e4848,
756 0x8d04, 0xffffffff, 0x00164745,
757 0x8c00, 0xffffffff, 0xe4000003,
758 0x8cf0, 0x1fffffff, 0x08e00410,
759 0x28350, 0xffffffff, 0x00000000,
760 0x9508, 0xffffffff, 0x00000002,
761 0x900c, 0xffffffff, 0x0017071f,
762 0x8c18, 0xffffffff, 0x10101060,
763 0x8c1c, 0xffffffff, 0x00001010
766 static const u32 barts_golden_registers[] =
768 0x5eb4, 0xffffffff, 0x00000002,
769 0x5e78, 0x8f311ff1, 0x001000f0,
770 0x3f90, 0xffff0000, 0xff000000,
771 0x9148, 0xffff0000, 0xff000000,
772 0x3f94, 0xffff0000, 0xff000000,
773 0x914c, 0xffff0000, 0xff000000,
774 0xc78, 0x00000080, 0x00000080,
775 0xbd4, 0x70073777, 0x00010001,
776 0xd02c, 0xbfffff1f, 0x08421000,
777 0xd0b8, 0x03773777, 0x02011003,
778 0x5bc0, 0x00200000, 0x50100000,
779 0x98f8, 0x33773777, 0x02011003,
780 0x98fc, 0xffffffff, 0x76543210,
781 0x7030, 0x31000311, 0x00000011,
782 0x2f48, 0x00000007, 0x02011003,
783 0x6b28, 0x00000010, 0x00000012,
784 0x7728, 0x00000010, 0x00000012,
785 0x10328, 0x00000010, 0x00000012,
786 0x10f28, 0x00000010, 0x00000012,
787 0x11b28, 0x00000010, 0x00000012,
788 0x12728, 0x00000010, 0x00000012,
789 0x240c, 0x000007ff, 0x00000380,
790 0x8a14, 0xf000001f, 0x00000007,
791 0x8b24, 0x3fff3fff, 0x00ff0fff,
792 0x8b10, 0x0000ff0f, 0x00000000,
793 0x28a4c, 0x07ffffff, 0x06000000,
794 0x10c, 0x00000001, 0x00010003,
795 0xa02c, 0xffffffff, 0x0000009b,
796 0x913c, 0x0000000f, 0x0100000a,
797 0x8d00, 0xffff7f7f, 0x100e4848,
798 0x8d04, 0x00ffffff, 0x00164745,
799 0x8c00, 0xfffc0003, 0xe4000003,
800 0x8c04, 0xf8ff00ff, 0x40600060,
801 0x8c08, 0x00ff00ff, 0x001c001c,
802 0x8cf0, 0x1fff1fff, 0x08e00620,
803 0x8c20, 0x0fff0fff, 0x00800080,
804 0x8c24, 0x0fff0fff, 0x00800080,
805 0x8c18, 0xffffffff, 0x20202078,
806 0x8c1c, 0x0000ffff, 0x00001010,
807 0x28350, 0x00000f01, 0x00000000,
808 0x9508, 0x3700001f, 0x00000002,
809 0x960c, 0xffffffff, 0x54763210,
810 0x88c4, 0x001f3ae3, 0x000000c2,
811 0x88d4, 0x0000001f, 0x00000010,
812 0x8974, 0xffffffff, 0x00000000
815 static const u32 turks_golden_registers[] =
817 0x5eb4, 0xffffffff, 0x00000002,
818 0x5e78, 0x8f311ff1, 0x001000f0,
819 0x8c8, 0x00003000, 0x00001070,
820 0x8cc, 0x000fffff, 0x00040035,
821 0x3f90, 0xffff0000, 0xfff00000,
822 0x9148, 0xffff0000, 0xfff00000,
823 0x3f94, 0xffff0000, 0xfff00000,
824 0x914c, 0xffff0000, 0xfff00000,
825 0xc78, 0x00000080, 0x00000080,
826 0xbd4, 0x00073007, 0x00010002,
827 0xd02c, 0xbfffff1f, 0x08421000,
828 0xd0b8, 0x03773777, 0x02010002,
829 0x5bc0, 0x00200000, 0x50100000,
830 0x98f8, 0x33773777, 0x00010002,
831 0x98fc, 0xffffffff, 0x33221100,
832 0x7030, 0x31000311, 0x00000011,
833 0x2f48, 0x33773777, 0x00010002,
834 0x6b28, 0x00000010, 0x00000012,
835 0x7728, 0x00000010, 0x00000012,
836 0x10328, 0x00000010, 0x00000012,
837 0x10f28, 0x00000010, 0x00000012,
838 0x11b28, 0x00000010, 0x00000012,
839 0x12728, 0x00000010, 0x00000012,
840 0x240c, 0x000007ff, 0x00000380,
841 0x8a14, 0xf000001f, 0x00000007,
842 0x8b24, 0x3fff3fff, 0x00ff0fff,
843 0x8b10, 0x0000ff0f, 0x00000000,
844 0x28a4c, 0x07ffffff, 0x06000000,
845 0x10c, 0x00000001, 0x00010003,
846 0xa02c, 0xffffffff, 0x0000009b,
847 0x913c, 0x0000000f, 0x0100000a,
848 0x8d00, 0xffff7f7f, 0x100e4848,
849 0x8d04, 0x00ffffff, 0x00164745,
850 0x8c00, 0xfffc0003, 0xe4000003,
851 0x8c04, 0xf8ff00ff, 0x40600060,
852 0x8c08, 0x00ff00ff, 0x001c001c,
853 0x8cf0, 0x1fff1fff, 0x08e00410,
854 0x8c20, 0x0fff0fff, 0x00800080,
855 0x8c24, 0x0fff0fff, 0x00800080,
856 0x8c18, 0xffffffff, 0x20202078,
857 0x8c1c, 0x0000ffff, 0x00001010,
858 0x28350, 0x00000f01, 0x00000000,
859 0x9508, 0x3700001f, 0x00000002,
860 0x960c, 0xffffffff, 0x54763210,
861 0x88c4, 0x001f3ae3, 0x000000c2,
862 0x88d4, 0x0000001f, 0x00000010,
863 0x8974, 0xffffffff, 0x00000000
866 static const u32 caicos_golden_registers[] =
868 0x5eb4, 0xffffffff, 0x00000002,
869 0x5e78, 0x8f311ff1, 0x001000f0,
870 0x8c8, 0x00003420, 0x00001450,
871 0x8cc, 0x000fffff, 0x00040035,
872 0x3f90, 0xffff0000, 0xfffc0000,
873 0x9148, 0xffff0000, 0xfffc0000,
874 0x3f94, 0xffff0000, 0xfffc0000,
875 0x914c, 0xffff0000, 0xfffc0000,
876 0xc78, 0x00000080, 0x00000080,
877 0xbd4, 0x00073007, 0x00010001,
878 0xd02c, 0xbfffff1f, 0x08421000,
879 0xd0b8, 0x03773777, 0x02010001,
880 0x5bc0, 0x00200000, 0x50100000,
881 0x98f8, 0x33773777, 0x02010001,
882 0x98fc, 0xffffffff, 0x33221100,
883 0x7030, 0x31000311, 0x00000011,
884 0x2f48, 0x33773777, 0x02010001,
885 0x6b28, 0x00000010, 0x00000012,
886 0x7728, 0x00000010, 0x00000012,
887 0x10328, 0x00000010, 0x00000012,
888 0x10f28, 0x00000010, 0x00000012,
889 0x11b28, 0x00000010, 0x00000012,
890 0x12728, 0x00000010, 0x00000012,
891 0x240c, 0x000007ff, 0x00000380,
892 0x8a14, 0xf000001f, 0x00000001,
893 0x8b24, 0x3fff3fff, 0x00ff0fff,
894 0x8b10, 0x0000ff0f, 0x00000000,
895 0x28a4c, 0x07ffffff, 0x06000000,
896 0x10c, 0x00000001, 0x00010003,
897 0xa02c, 0xffffffff, 0x0000009b,
898 0x913c, 0x0000000f, 0x0100000a,
899 0x8d00, 0xffff7f7f, 0x100e4848,
900 0x8d04, 0x00ffffff, 0x00164745,
901 0x8c00, 0xfffc0003, 0xe4000003,
902 0x8c04, 0xf8ff00ff, 0x40600060,
903 0x8c08, 0x00ff00ff, 0x001c001c,
904 0x8cf0, 0x1fff1fff, 0x08e00410,
905 0x8c20, 0x0fff0fff, 0x00800080,
906 0x8c24, 0x0fff0fff, 0x00800080,
907 0x8c18, 0xffffffff, 0x20202078,
908 0x8c1c, 0x0000ffff, 0x00001010,
909 0x28350, 0x00000f01, 0x00000000,
910 0x9508, 0x3700001f, 0x00000002,
911 0x960c, 0xffffffff, 0x54763210,
912 0x88c4, 0x001f3ae3, 0x000000c2,
913 0x88d4, 0x0000001f, 0x00000010,
914 0x8974, 0xffffffff, 0x00000000
917 static void evergreen_init_golden_registers(struct radeon_device *rdev)
919 switch (rdev->family) {
922 radeon_program_register_sequence(rdev,
923 evergreen_golden_registers,
924 (const u32)ARRAY_SIZE(evergreen_golden_registers));
925 radeon_program_register_sequence(rdev,
926 evergreen_golden_registers2,
927 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
928 radeon_program_register_sequence(rdev,
930 (const u32)ARRAY_SIZE(cypress_mgcg_init));
933 radeon_program_register_sequence(rdev,
934 evergreen_golden_registers,
935 (const u32)ARRAY_SIZE(evergreen_golden_registers));
936 radeon_program_register_sequence(rdev,
937 evergreen_golden_registers2,
938 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
939 radeon_program_register_sequence(rdev,
941 (const u32)ARRAY_SIZE(juniper_mgcg_init));
944 radeon_program_register_sequence(rdev,
945 evergreen_golden_registers,
946 (const u32)ARRAY_SIZE(evergreen_golden_registers));
947 radeon_program_register_sequence(rdev,
948 evergreen_golden_registers2,
949 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
950 radeon_program_register_sequence(rdev,
952 (const u32)ARRAY_SIZE(redwood_mgcg_init));
955 radeon_program_register_sequence(rdev,
956 cedar_golden_registers,
957 (const u32)ARRAY_SIZE(cedar_golden_registers));
958 radeon_program_register_sequence(rdev,
959 evergreen_golden_registers2,
960 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
961 radeon_program_register_sequence(rdev,
963 (const u32)ARRAY_SIZE(cedar_mgcg_init));
966 radeon_program_register_sequence(rdev,
967 wrestler_golden_registers,
968 (const u32)ARRAY_SIZE(wrestler_golden_registers));
971 radeon_program_register_sequence(rdev,
972 supersumo_golden_registers,
973 (const u32)ARRAY_SIZE(supersumo_golden_registers));
976 radeon_program_register_sequence(rdev,
977 supersumo_golden_registers,
978 (const u32)ARRAY_SIZE(supersumo_golden_registers));
979 radeon_program_register_sequence(rdev,
980 sumo_golden_registers,
981 (const u32)ARRAY_SIZE(sumo_golden_registers));
984 radeon_program_register_sequence(rdev,
985 barts_golden_registers,
986 (const u32)ARRAY_SIZE(barts_golden_registers));
989 radeon_program_register_sequence(rdev,
990 turks_golden_registers,
991 (const u32)ARRAY_SIZE(turks_golden_registers));
994 radeon_program_register_sequence(rdev,
995 caicos_golden_registers,
996 (const u32)ARRAY_SIZE(caicos_golden_registers));
1003 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1004 unsigned *bankh, unsigned *mtaspect,
1005 unsigned *tile_split)
1007 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1008 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1009 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1010 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1013 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1014 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1015 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1016 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1020 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1021 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1022 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1023 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1025 switch (*mtaspect) {
1027 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1028 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1029 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1030 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1034 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1035 u32 cntl_reg, u32 status_reg)
1038 struct atom_clock_dividers dividers;
1040 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1041 clock, false, ÷rs);
1045 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1047 for (i = 0; i < 100; i++) {
1048 if (RREG32(status_reg) & DCLK_STATUS)
1058 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1061 u32 cg_scratch = RREG32(CG_SCRATCH1);
1063 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1066 cg_scratch &= 0xffff0000;
1067 cg_scratch |= vclk / 100; /* Mhz */
1069 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1072 cg_scratch &= 0x0000ffff;
1073 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1076 WREG32(CG_SCRATCH1, cg_scratch);
1081 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1083 /* start off with something large */
1084 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1087 /* bypass vclk and dclk with bclk */
1088 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1089 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1090 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1092 /* put PLL in bypass mode */
1093 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1095 if (!vclk || !dclk) {
1096 /* keep the Bypass mode, put PLL to sleep */
1097 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1101 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1102 16384, 0x03FFFFFF, 0, 128, 5,
1103 &fb_div, &vclk_div, &dclk_div);
1107 /* set VCO_MODE to 1 */
1108 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1110 /* toggle UPLL_SLEEP to 1 then back to 0 */
1111 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1112 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1114 /* deassert UPLL_RESET */
1115 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1119 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1123 /* assert UPLL_RESET again */
1124 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1126 /* disable spread spectrum. */
1127 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1129 /* set feedback divider */
1130 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1132 /* set ref divider to 0 */
1133 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1135 if (fb_div < 307200)
1136 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1138 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1140 /* set PDIV_A and PDIV_B */
1141 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1142 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1143 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1145 /* give the PLL some time to settle */
1148 /* deassert PLL_RESET */
1149 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1153 /* switch from bypass mode to normal mode */
1154 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1156 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1160 /* switch VCLK and DCLK selection */
1161 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1162 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1163 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1170 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1175 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
1179 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1181 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1182 * to avoid hangs or perfomance issues
1184 if ((v == 0) || (v == 6) || (v == 7)) {
1185 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1187 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
1191 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1193 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1199 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1203 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1204 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1213 * dce4_wait_for_vblank - vblank wait asic callback.
1215 * @rdev: radeon_device pointer
1216 * @crtc: crtc to wait for vblank on
1218 * Wait for vblank on the requested crtc (evergreen+).
1220 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1224 if (crtc >= rdev->num_crtc)
1227 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1230 /* depending on when we hit vblank, we may be close to active; if so,
1231 * wait for another frame.
1233 while (dce4_is_in_vblank(rdev, crtc)) {
1234 if (i++ % 100 == 0) {
1235 if (!dce4_is_counter_moving(rdev, crtc))
1240 while (!dce4_is_in_vblank(rdev, crtc)) {
1241 if (i++ % 100 == 0) {
1242 if (!dce4_is_counter_moving(rdev, crtc))
1249 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1251 * @rdev: radeon_device pointer
1252 * @crtc: crtc to prepare for pageflip on
1254 * Pre-pageflip callback (evergreen+).
1255 * Enables the pageflip irq (vblank irq).
1257 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1259 /* enable the pflip int */
1260 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1264 * evergreen_post_page_flip - pos-pageflip callback.
1266 * @rdev: radeon_device pointer
1267 * @crtc: crtc to cleanup pageflip on
1269 * Post-pageflip callback (evergreen+).
1270 * Disables the pageflip irq (vblank irq).
1272 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1274 /* disable the pflip int */
1275 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1279 * evergreen_page_flip - pageflip callback.
1281 * @rdev: radeon_device pointer
1282 * @crtc_id: crtc to cleanup pageflip on
1283 * @crtc_base: new address of the crtc (GPU MC address)
1285 * Does the actual pageflip (evergreen+).
1286 * During vblank we take the crtc lock and wait for the update_pending
1287 * bit to go high, when it does, we release the lock, and allow the
1288 * double buffered update to take place.
1289 * Returns the current update pending status.
1291 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1293 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1294 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1297 /* Lock the graphics update lock */
1298 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1299 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1301 /* update the scanout addresses */
1302 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1303 upper_32_bits(crtc_base));
1304 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1307 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1308 upper_32_bits(crtc_base));
1309 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1312 /* Wait for update_pending to go high. */
1313 for (i = 0; i < rdev->usec_timeout; i++) {
1314 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1318 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1320 /* Unlock the lock, so double-buffering can take place inside vblank */
1321 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1322 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1324 /* Return current update_pending status: */
1325 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1328 /* get temperature in millidegrees */
1329 int evergreen_get_temp(struct radeon_device *rdev)
1332 int actual_temp = 0;
1334 if (rdev->family == CHIP_JUNIPER) {
1335 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1337 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1340 if (toffset & 0x100)
1341 actual_temp = temp / 2 - (0x200 - toffset);
1343 actual_temp = temp / 2 + toffset;
1345 actual_temp = actual_temp * 1000;
1348 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1353 else if (temp & 0x200)
1355 else if (temp & 0x100) {
1356 actual_temp = temp & 0x1ff;
1357 actual_temp |= ~0x1ff;
1359 actual_temp = temp & 0xff;
1361 actual_temp = (actual_temp * 1000) / 2;
1367 int sumo_get_temp(struct radeon_device *rdev)
1369 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1370 int actual_temp = temp - 49;
1372 return actual_temp * 1000;
1376 * sumo_pm_init_profile - Initialize power profiles callback.
1378 * @rdev: radeon_device pointer
1380 * Initialize the power states used in profile mode
1381 * (sumo, trinity, SI).
1382 * Used for profile mode only.
1384 void sumo_pm_init_profile(struct radeon_device *rdev)
1389 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1390 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1391 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1392 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1395 if (rdev->flags & RADEON_IS_MOBILITY)
1396 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1398 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1400 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1401 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1402 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1403 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1405 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1406 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1407 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1408 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1410 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1411 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1412 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1413 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1416 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1417 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1418 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1421 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1422 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1423 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1424 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1425 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1426 rdev->pm.power_state[idx].num_clock_modes - 1;
1428 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1429 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1430 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1431 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1432 rdev->pm.power_state[idx].num_clock_modes - 1;
1436 * btc_pm_init_profile - Initialize power profiles callback.
1438 * @rdev: radeon_device pointer
1440 * Initialize the power states used in profile mode
1442 * Used for profile mode only.
1444 void btc_pm_init_profile(struct radeon_device *rdev)
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1450 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1451 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1452 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1453 /* starting with BTC, there is one state that is used for both
1454 * MH and SH. Difference is that we always use the high clock index for
1457 if (rdev->flags & RADEON_IS_MOBILITY)
1458 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1460 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1462 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1463 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1464 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1465 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1469 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1470 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1472 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1473 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1474 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1475 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1477 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1478 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1479 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1480 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1483 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1484 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1485 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1488 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1489 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1490 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1494 * evergreen_pm_misc - set additional pm hw parameters callback.
1496 * @rdev: radeon_device pointer
1498 * Set non-clock parameters associated with a power state
1499 * (voltage, etc.) (evergreen+).
1501 void evergreen_pm_misc(struct radeon_device *rdev)
1503 int req_ps_idx = rdev->pm.requested_power_state_index;
1504 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1505 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1506 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1508 if (voltage->type == VOLTAGE_SW) {
1509 /* 0xff0x are flags rather then an actual voltage */
1510 if ((voltage->voltage & 0xff00) == 0xff00)
1512 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1513 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1514 rdev->pm.current_vddc = voltage->voltage;
1515 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1518 /* starting with BTC, there is one state that is used for both
1519 * MH and SH. Difference is that we always use the high clock index for
1522 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1523 (rdev->family >= CHIP_BARTS) &&
1524 rdev->pm.active_crtc_count &&
1525 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1526 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1527 voltage = &rdev->pm.power_state[req_ps_idx].
1528 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1530 /* 0xff0x are flags rather then an actual voltage */
1531 if ((voltage->vddci & 0xff00) == 0xff00)
1533 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1534 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1535 rdev->pm.current_vddci = voltage->vddci;
1536 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1542 * evergreen_pm_prepare - pre-power state change callback.
1544 * @rdev: radeon_device pointer
1546 * Prepare for a power state change (evergreen+).
1548 void evergreen_pm_prepare(struct radeon_device *rdev)
1550 struct drm_device *ddev = rdev->ddev;
1551 struct drm_crtc *crtc;
1552 struct radeon_crtc *radeon_crtc;
1555 /* disable any active CRTCs */
1556 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1557 radeon_crtc = to_radeon_crtc(crtc);
1558 if (radeon_crtc->enabled) {
1559 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1560 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1561 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1567 * evergreen_pm_finish - post-power state change callback.
1569 * @rdev: radeon_device pointer
1571 * Clean up after a power state change (evergreen+).
1573 void evergreen_pm_finish(struct radeon_device *rdev)
1575 struct drm_device *ddev = rdev->ddev;
1576 struct drm_crtc *crtc;
1577 struct radeon_crtc *radeon_crtc;
1580 /* enable any active CRTCs */
1581 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1582 radeon_crtc = to_radeon_crtc(crtc);
1583 if (radeon_crtc->enabled) {
1584 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1585 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1586 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1592 * evergreen_hpd_sense - hpd sense callback.
1594 * @rdev: radeon_device pointer
1595 * @hpd: hpd (hotplug detect) pin
1597 * Checks if a digital monitor is connected (evergreen+).
1598 * Returns true if connected, false if not connected.
1600 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1602 bool connected = false;
1606 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1610 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1614 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1618 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1622 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1626 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1637 * evergreen_hpd_set_polarity - hpd set polarity callback.
1639 * @rdev: radeon_device pointer
1640 * @hpd: hpd (hotplug detect) pin
1642 * Set the polarity of the hpd pin (evergreen+).
1644 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1645 enum radeon_hpd_id hpd)
1648 bool connected = evergreen_hpd_sense(rdev, hpd);
1652 tmp = RREG32(DC_HPD1_INT_CONTROL);
1654 tmp &= ~DC_HPDx_INT_POLARITY;
1656 tmp |= DC_HPDx_INT_POLARITY;
1657 WREG32(DC_HPD1_INT_CONTROL, tmp);
1660 tmp = RREG32(DC_HPD2_INT_CONTROL);
1662 tmp &= ~DC_HPDx_INT_POLARITY;
1664 tmp |= DC_HPDx_INT_POLARITY;
1665 WREG32(DC_HPD2_INT_CONTROL, tmp);
1668 tmp = RREG32(DC_HPD3_INT_CONTROL);
1670 tmp &= ~DC_HPDx_INT_POLARITY;
1672 tmp |= DC_HPDx_INT_POLARITY;
1673 WREG32(DC_HPD3_INT_CONTROL, tmp);
1676 tmp = RREG32(DC_HPD4_INT_CONTROL);
1678 tmp &= ~DC_HPDx_INT_POLARITY;
1680 tmp |= DC_HPDx_INT_POLARITY;
1681 WREG32(DC_HPD4_INT_CONTROL, tmp);
1684 tmp = RREG32(DC_HPD5_INT_CONTROL);
1686 tmp &= ~DC_HPDx_INT_POLARITY;
1688 tmp |= DC_HPDx_INT_POLARITY;
1689 WREG32(DC_HPD5_INT_CONTROL, tmp);
1692 tmp = RREG32(DC_HPD6_INT_CONTROL);
1694 tmp &= ~DC_HPDx_INT_POLARITY;
1696 tmp |= DC_HPDx_INT_POLARITY;
1697 WREG32(DC_HPD6_INT_CONTROL, tmp);
1705 * evergreen_hpd_init - hpd setup callback.
1707 * @rdev: radeon_device pointer
1709 * Setup the hpd pins used by the card (evergreen+).
1710 * Enable the pin, set the polarity, and enable the hpd interrupts.
1712 void evergreen_hpd_init(struct radeon_device *rdev)
1714 struct drm_device *dev = rdev->ddev;
1715 struct drm_connector *connector;
1716 unsigned enabled = 0;
1717 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1718 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1720 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1721 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1723 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1724 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1725 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1726 * aux dp channel on imac and help (but not completely fix)
1727 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1728 * also avoid interrupt storms during dpms.
1732 switch (radeon_connector->hpd.hpd) {
1734 WREG32(DC_HPD1_CONTROL, tmp);
1737 WREG32(DC_HPD2_CONTROL, tmp);
1740 WREG32(DC_HPD3_CONTROL, tmp);
1743 WREG32(DC_HPD4_CONTROL, tmp);
1746 WREG32(DC_HPD5_CONTROL, tmp);
1749 WREG32(DC_HPD6_CONTROL, tmp);
1754 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1755 enabled |= 1 << radeon_connector->hpd.hpd;
1757 radeon_irq_kms_enable_hpd(rdev, enabled);
1761 * evergreen_hpd_fini - hpd tear down callback.
1763 * @rdev: radeon_device pointer
1765 * Tear down the hpd pins used by the card (evergreen+).
1766 * Disable the hpd interrupts.
1768 void evergreen_hpd_fini(struct radeon_device *rdev)
1770 struct drm_device *dev = rdev->ddev;
1771 struct drm_connector *connector;
1772 unsigned disabled = 0;
1774 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1775 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1776 switch (radeon_connector->hpd.hpd) {
1778 WREG32(DC_HPD1_CONTROL, 0);
1781 WREG32(DC_HPD2_CONTROL, 0);
1784 WREG32(DC_HPD3_CONTROL, 0);
1787 WREG32(DC_HPD4_CONTROL, 0);
1790 WREG32(DC_HPD5_CONTROL, 0);
1793 WREG32(DC_HPD6_CONTROL, 0);
1798 disabled |= 1 << radeon_connector->hpd.hpd;
1800 radeon_irq_kms_disable_hpd(rdev, disabled);
1803 /* watermark setup */
1805 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1806 struct radeon_crtc *radeon_crtc,
1807 struct drm_display_mode *mode,
1808 struct drm_display_mode *other_mode)
1813 * There are 3 line buffers, each one shared by 2 display controllers.
1814 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1815 * the display controllers. The paritioning is done via one of four
1816 * preset allocations specified in bits 2:0:
1817 * first display controller
1818 * 0 - first half of lb (3840 * 2)
1819 * 1 - first 3/4 of lb (5760 * 2)
1820 * 2 - whole lb (7680 * 2), other crtc must be disabled
1821 * 3 - first 1/4 of lb (1920 * 2)
1822 * second display controller
1823 * 4 - second half of lb (3840 * 2)
1824 * 5 - second 3/4 of lb (5760 * 2)
1825 * 6 - whole lb (7680 * 2), other crtc must be disabled
1826 * 7 - last 1/4 of lb (1920 * 2)
1828 /* this can get tricky if we have two large displays on a paired group
1829 * of crtcs. Ideally for multiple large displays we'd assign them to
1830 * non-linked crtcs for maximum line buffer allocation.
1832 if (radeon_crtc->base.enabled && mode) {
1836 tmp = 2; /* whole */
1840 /* second controller of the pair uses second half of the lb */
1841 if (radeon_crtc->crtc_id % 2)
1843 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1845 if (radeon_crtc->base.enabled && mode) {
1850 if (ASIC_IS_DCE5(rdev))
1856 if (ASIC_IS_DCE5(rdev))
1862 if (ASIC_IS_DCE5(rdev))
1868 if (ASIC_IS_DCE5(rdev))
1875 /* controller not enabled, so no lb used */
1879 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1881 u32 tmp = RREG32(MC_SHARED_CHMAP);
1883 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1896 struct evergreen_wm_params {
1897 u32 dram_channels; /* number of dram channels */
1898 u32 yclk; /* bandwidth per dram data pin in kHz */
1899 u32 sclk; /* engine clock in kHz */
1900 u32 disp_clk; /* display clock in kHz */
1901 u32 src_width; /* viewport width */
1902 u32 active_time; /* active display time in ns */
1903 u32 blank_time; /* blank time in ns */
1904 bool interlaced; /* mode is interlaced */
1905 fixed20_12 vsc; /* vertical scale ratio */
1906 u32 num_heads; /* number of active crtcs */
1907 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1908 u32 lb_size; /* line buffer allocated to pipe */
1909 u32 vtaps; /* vertical scaler taps */
1912 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1914 /* Calculate DRAM Bandwidth and the part allocated to display. */
1915 fixed20_12 dram_efficiency; /* 0.7 */
1916 fixed20_12 yclk, dram_channels, bandwidth;
1919 a.full = dfixed_const(1000);
1920 yclk.full = dfixed_const(wm->yclk);
1921 yclk.full = dfixed_div(yclk, a);
1922 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1923 a.full = dfixed_const(10);
1924 dram_efficiency.full = dfixed_const(7);
1925 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1926 bandwidth.full = dfixed_mul(dram_channels, yclk);
1927 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1929 return dfixed_trunc(bandwidth);
1932 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1934 /* Calculate DRAM Bandwidth and the part allocated to display. */
1935 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1936 fixed20_12 yclk, dram_channels, bandwidth;
1939 a.full = dfixed_const(1000);
1940 yclk.full = dfixed_const(wm->yclk);
1941 yclk.full = dfixed_div(yclk, a);
1942 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1943 a.full = dfixed_const(10);
1944 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1945 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1946 bandwidth.full = dfixed_mul(dram_channels, yclk);
1947 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1949 return dfixed_trunc(bandwidth);
1952 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1954 /* Calculate the display Data return Bandwidth */
1955 fixed20_12 return_efficiency; /* 0.8 */
1956 fixed20_12 sclk, bandwidth;
1959 a.full = dfixed_const(1000);
1960 sclk.full = dfixed_const(wm->sclk);
1961 sclk.full = dfixed_div(sclk, a);
1962 a.full = dfixed_const(10);
1963 return_efficiency.full = dfixed_const(8);
1964 return_efficiency.full = dfixed_div(return_efficiency, a);
1965 a.full = dfixed_const(32);
1966 bandwidth.full = dfixed_mul(a, sclk);
1967 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1969 return dfixed_trunc(bandwidth);
1972 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1974 /* Calculate the DMIF Request Bandwidth */
1975 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1976 fixed20_12 disp_clk, bandwidth;
1979 a.full = dfixed_const(1000);
1980 disp_clk.full = dfixed_const(wm->disp_clk);
1981 disp_clk.full = dfixed_div(disp_clk, a);
1982 a.full = dfixed_const(10);
1983 disp_clk_request_efficiency.full = dfixed_const(8);
1984 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1985 a.full = dfixed_const(32);
1986 bandwidth.full = dfixed_mul(a, disp_clk);
1987 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1989 return dfixed_trunc(bandwidth);
1992 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1994 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1995 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1996 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1997 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1999 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2002 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2004 /* Calculate the display mode Average Bandwidth
2005 * DisplayMode should contain the source and destination dimensions,
2009 fixed20_12 line_time;
2010 fixed20_12 src_width;
2011 fixed20_12 bandwidth;
2014 a.full = dfixed_const(1000);
2015 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2016 line_time.full = dfixed_div(line_time, a);
2017 bpp.full = dfixed_const(wm->bytes_per_pixel);
2018 src_width.full = dfixed_const(wm->src_width);
2019 bandwidth.full = dfixed_mul(src_width, bpp);
2020 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2021 bandwidth.full = dfixed_div(bandwidth, line_time);
2023 return dfixed_trunc(bandwidth);
2026 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2028 /* First calcualte the latency in ns */
2029 u32 mc_latency = 2000; /* 2000 ns. */
2030 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2031 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2032 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2033 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2034 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2035 (wm->num_heads * cursor_line_pair_return_time);
2036 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2037 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2040 if (wm->num_heads == 0)
2043 a.full = dfixed_const(2);
2044 b.full = dfixed_const(1);
2045 if ((wm->vsc.full > a.full) ||
2046 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2048 ((wm->vsc.full >= a.full) && wm->interlaced))
2049 max_src_lines_per_dst_line = 4;
2051 max_src_lines_per_dst_line = 2;
2053 a.full = dfixed_const(available_bandwidth);
2054 b.full = dfixed_const(wm->num_heads);
2055 a.full = dfixed_div(a, b);
2057 b.full = dfixed_const(1000);
2058 c.full = dfixed_const(wm->disp_clk);
2059 b.full = dfixed_div(c, b);
2060 c.full = dfixed_const(wm->bytes_per_pixel);
2061 b.full = dfixed_mul(b, c);
2063 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2065 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2066 b.full = dfixed_const(1000);
2067 c.full = dfixed_const(lb_fill_bw);
2068 b.full = dfixed_div(c, b);
2069 a.full = dfixed_div(a, b);
2070 line_fill_time = dfixed_trunc(a);
2072 if (line_fill_time < wm->active_time)
2075 return latency + (line_fill_time - wm->active_time);
2079 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2081 if (evergreen_average_bandwidth(wm) <=
2082 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2088 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2090 if (evergreen_average_bandwidth(wm) <=
2091 (evergreen_available_bandwidth(wm) / wm->num_heads))
2097 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2099 u32 lb_partitions = wm->lb_size / wm->src_width;
2100 u32 line_time = wm->active_time + wm->blank_time;
2101 u32 latency_tolerant_lines;
2105 a.full = dfixed_const(1);
2106 if (wm->vsc.full > a.full)
2107 latency_tolerant_lines = 1;
2109 if (lb_partitions <= (wm->vtaps + 1))
2110 latency_tolerant_lines = 1;
2112 latency_tolerant_lines = 2;
2115 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2117 if (evergreen_latency_watermark(wm) <= latency_hiding)
2123 static void evergreen_program_watermarks(struct radeon_device *rdev,
2124 struct radeon_crtc *radeon_crtc,
2125 u32 lb_size, u32 num_heads)
2127 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2128 struct evergreen_wm_params wm_low, wm_high;
2132 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2133 u32 priority_a_mark = 0, priority_b_mark = 0;
2134 u32 priority_a_cnt = PRIORITY_OFF;
2135 u32 priority_b_cnt = PRIORITY_OFF;
2136 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2137 u32 tmp, arb_control3;
2140 if (radeon_crtc->base.enabled && num_heads && mode) {
2141 pixel_period = 1000000 / (u32)mode->clock;
2142 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2145 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2147 /* watermark for high clocks */
2148 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2150 radeon_dpm_get_mclk(rdev, false) * 10;
2152 radeon_dpm_get_sclk(rdev, false) * 10;
2154 wm_high.yclk = rdev->pm.current_mclk * 10;
2155 wm_high.sclk = rdev->pm.current_sclk * 10;
2158 wm_high.disp_clk = mode->clock;
2159 wm_high.src_width = mode->crtc_hdisplay;
2160 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2161 wm_high.blank_time = line_time - wm_high.active_time;
2162 wm_high.interlaced = false;
2163 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2164 wm_high.interlaced = true;
2165 wm_high.vsc = radeon_crtc->vsc;
2167 if (radeon_crtc->rmx_type != RMX_OFF)
2169 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2170 wm_high.lb_size = lb_size;
2171 wm_high.dram_channels = dram_channels;
2172 wm_high.num_heads = num_heads;
2174 /* watermark for low clocks */
2175 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2177 radeon_dpm_get_mclk(rdev, true) * 10;
2179 radeon_dpm_get_sclk(rdev, true) * 10;
2181 wm_low.yclk = rdev->pm.current_mclk * 10;
2182 wm_low.sclk = rdev->pm.current_sclk * 10;
2185 wm_low.disp_clk = mode->clock;
2186 wm_low.src_width = mode->crtc_hdisplay;
2187 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2188 wm_low.blank_time = line_time - wm_low.active_time;
2189 wm_low.interlaced = false;
2190 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2191 wm_low.interlaced = true;
2192 wm_low.vsc = radeon_crtc->vsc;
2194 if (radeon_crtc->rmx_type != RMX_OFF)
2196 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2197 wm_low.lb_size = lb_size;
2198 wm_low.dram_channels = dram_channels;
2199 wm_low.num_heads = num_heads;
2201 /* set for high clocks */
2202 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2203 /* set for low clocks */
2204 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2206 /* possibly force display priority to high */
2207 /* should really do this at mode validation time... */
2208 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2209 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2210 !evergreen_check_latency_hiding(&wm_high) ||
2211 (rdev->disp_priority == 2)) {
2212 DRM_DEBUG_KMS("force priority a to high\n");
2213 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2215 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2216 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2217 !evergreen_check_latency_hiding(&wm_low) ||
2218 (rdev->disp_priority == 2)) {
2219 DRM_DEBUG_KMS("force priority b to high\n");
2220 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2223 a.full = dfixed_const(1000);
2224 b.full = dfixed_const(mode->clock);
2225 b.full = dfixed_div(b, a);
2226 c.full = dfixed_const(latency_watermark_a);
2227 c.full = dfixed_mul(c, b);
2228 c.full = dfixed_mul(c, radeon_crtc->hsc);
2229 c.full = dfixed_div(c, a);
2230 a.full = dfixed_const(16);
2231 c.full = dfixed_div(c, a);
2232 priority_a_mark = dfixed_trunc(c);
2233 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2235 a.full = dfixed_const(1000);
2236 b.full = dfixed_const(mode->clock);
2237 b.full = dfixed_div(b, a);
2238 c.full = dfixed_const(latency_watermark_b);
2239 c.full = dfixed_mul(c, b);
2240 c.full = dfixed_mul(c, radeon_crtc->hsc);
2241 c.full = dfixed_div(c, a);
2242 a.full = dfixed_const(16);
2243 c.full = dfixed_div(c, a);
2244 priority_b_mark = dfixed_trunc(c);
2245 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2249 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2251 tmp &= ~LATENCY_WATERMARK_MASK(3);
2252 tmp |= LATENCY_WATERMARK_MASK(1);
2253 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2254 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2255 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2256 LATENCY_HIGH_WATERMARK(line_time)));
2258 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2259 tmp &= ~LATENCY_WATERMARK_MASK(3);
2260 tmp |= LATENCY_WATERMARK_MASK(2);
2261 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2262 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2263 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2264 LATENCY_HIGH_WATERMARK(line_time)));
2265 /* restore original selection */
2266 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2268 /* write the priority marks */
2269 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2270 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2272 /* save values for DPM */
2273 radeon_crtc->line_time = line_time;
2274 radeon_crtc->wm_high = latency_watermark_a;
2275 radeon_crtc->wm_low = latency_watermark_b;
2279 * evergreen_bandwidth_update - update display watermarks callback.
2281 * @rdev: radeon_device pointer
2283 * Update the display watermarks based on the requested mode(s)
2286 void evergreen_bandwidth_update(struct radeon_device *rdev)
2288 struct drm_display_mode *mode0 = NULL;
2289 struct drm_display_mode *mode1 = NULL;
2290 u32 num_heads = 0, lb_size;
2293 radeon_update_display_priority(rdev);
2295 for (i = 0; i < rdev->num_crtc; i++) {
2296 if (rdev->mode_info.crtcs[i]->base.enabled)
2299 for (i = 0; i < rdev->num_crtc; i += 2) {
2300 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2301 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2302 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2303 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2304 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2305 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2310 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2312 * @rdev: radeon_device pointer
2314 * Wait for the MC (memory controller) to be idle.
2316 * Returns 0 if the MC is idle, -1 if not.
2318 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2323 for (i = 0; i < rdev->usec_timeout; i++) {
2324 /* read MC_STATUS */
2325 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2336 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2341 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2343 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2344 for (i = 0; i < rdev->usec_timeout; i++) {
2345 /* read MC_STATUS */
2346 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2347 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2349 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2359 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2364 if (rdev->gart.robj == NULL) {
2365 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2368 r = radeon_gart_table_vram_pin(rdev);
2371 radeon_gart_restore(rdev);
2372 /* Setup L2 cache */
2373 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2374 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2375 EFFECTIVE_L2_QUEUE_SIZE(7));
2376 WREG32(VM_L2_CNTL2, 0);
2377 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2378 /* Setup TLB control */
2379 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2380 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2381 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2382 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2383 if (rdev->flags & RADEON_IS_IGP) {
2384 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2385 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2386 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2388 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2389 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2390 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2391 if ((rdev->family == CHIP_JUNIPER) ||
2392 (rdev->family == CHIP_CYPRESS) ||
2393 (rdev->family == CHIP_HEMLOCK) ||
2394 (rdev->family == CHIP_BARTS))
2395 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2397 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2398 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2399 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2400 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2401 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2402 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2403 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2404 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2405 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2406 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2407 (u32)(rdev->dummy_page.addr >> 12));
2408 WREG32(VM_CONTEXT1_CNTL, 0);
2410 evergreen_pcie_gart_tlb_flush(rdev);
2411 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2412 (unsigned)(rdev->mc.gtt_size >> 20),
2413 (unsigned long long)rdev->gart.table_addr);
2414 rdev->gart.ready = true;
2418 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2422 /* Disable all tables */
2423 WREG32(VM_CONTEXT0_CNTL, 0);
2424 WREG32(VM_CONTEXT1_CNTL, 0);
2426 /* Setup L2 cache */
2427 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2428 EFFECTIVE_L2_QUEUE_SIZE(7));
2429 WREG32(VM_L2_CNTL2, 0);
2430 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2431 /* Setup TLB control */
2432 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2433 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2434 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2435 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2436 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2437 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2438 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2439 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2440 radeon_gart_table_vram_unpin(rdev);
2443 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2445 evergreen_pcie_gart_disable(rdev);
2446 radeon_gart_table_vram_free(rdev);
2447 radeon_gart_fini(rdev);
2451 static void evergreen_agp_enable(struct radeon_device *rdev)
2455 /* Setup L2 cache */
2456 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2457 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2458 EFFECTIVE_L2_QUEUE_SIZE(7));
2459 WREG32(VM_L2_CNTL2, 0);
2460 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2461 /* Setup TLB control */
2462 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2463 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2464 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2465 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2466 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2467 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2468 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2469 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2470 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2471 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2472 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2473 WREG32(VM_CONTEXT0_CNTL, 0);
2474 WREG32(VM_CONTEXT1_CNTL, 0);
2477 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2479 u32 crtc_enabled, tmp, frame_count, blackout;
2482 if (!ASIC_IS_NODCE(rdev)) {
2483 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2484 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2486 /* disable VGA render */
2487 WREG32(VGA_RENDER_CONTROL, 0);
2489 /* blank the display controllers */
2490 for (i = 0; i < rdev->num_crtc; i++) {
2491 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2493 save->crtc_enabled[i] = true;
2494 if (ASIC_IS_DCE6(rdev)) {
2495 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2496 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2497 radeon_wait_for_vblank(rdev, i);
2498 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2499 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2500 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2503 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2504 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2505 radeon_wait_for_vblank(rdev, i);
2506 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2507 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2508 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2509 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2512 /* wait for the next frame */
2513 frame_count = radeon_get_vblank_counter(rdev, i);
2514 for (j = 0; j < rdev->usec_timeout; j++) {
2515 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2520 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2521 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2522 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2523 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2524 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2525 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2526 save->crtc_enabled[i] = false;
2529 save->crtc_enabled[i] = false;
2533 radeon_mc_wait_for_idle(rdev);
2535 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2536 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2537 /* Block CPU access */
2538 WREG32(BIF_FB_EN, 0);
2539 /* blackout the MC */
2540 blackout &= ~BLACKOUT_MODE_MASK;
2541 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2543 /* wait for the MC to settle */
2546 /* lock double buffered regs */
2547 for (i = 0; i < rdev->num_crtc; i++) {
2548 if (save->crtc_enabled[i]) {
2549 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2550 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2551 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2552 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2554 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2557 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2563 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2565 u32 tmp, frame_count;
2568 /* update crtc base addresses */
2569 for (i = 0; i < rdev->num_crtc; i++) {
2570 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2571 upper_32_bits(rdev->mc.vram_start));
2572 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2573 upper_32_bits(rdev->mc.vram_start));
2574 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2575 (u32)rdev->mc.vram_start);
2576 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2577 (u32)rdev->mc.vram_start);
2580 if (!ASIC_IS_NODCE(rdev)) {
2581 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2582 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2585 /* unlock regs and wait for update */
2586 for (i = 0; i < rdev->num_crtc; i++) {
2587 if (save->crtc_enabled[i]) {
2588 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2589 if ((tmp & 0x3) != 0) {
2591 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2593 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2594 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2595 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2596 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2598 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2601 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2603 for (j = 0; j < rdev->usec_timeout; j++) {
2604 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2605 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2612 /* unblackout the MC */
2613 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2614 tmp &= ~BLACKOUT_MODE_MASK;
2615 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2616 /* allow CPU access */
2617 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2619 for (i = 0; i < rdev->num_crtc; i++) {
2620 if (save->crtc_enabled[i]) {
2621 if (ASIC_IS_DCE6(rdev)) {
2622 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2623 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2624 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2625 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2626 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2628 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2629 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2630 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2631 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2632 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2634 /* wait for the next frame */
2635 frame_count = radeon_get_vblank_counter(rdev, i);
2636 for (j = 0; j < rdev->usec_timeout; j++) {
2637 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2643 if (!ASIC_IS_NODCE(rdev)) {
2644 /* Unlock vga access */
2645 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2647 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2651 void evergreen_mc_program(struct radeon_device *rdev)
2653 struct evergreen_mc_save save;
2657 /* Initialize HDP */
2658 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2659 WREG32((0x2c14 + j), 0x00000000);
2660 WREG32((0x2c18 + j), 0x00000000);
2661 WREG32((0x2c1c + j), 0x00000000);
2662 WREG32((0x2c20 + j), 0x00000000);
2663 WREG32((0x2c24 + j), 0x00000000);
2665 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2667 evergreen_mc_stop(rdev, &save);
2668 if (evergreen_mc_wait_for_idle(rdev)) {
2669 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2671 /* Lockout access through VGA aperture*/
2672 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2673 /* Update configuration */
2674 if (rdev->flags & RADEON_IS_AGP) {
2675 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2676 /* VRAM before AGP */
2677 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2678 rdev->mc.vram_start >> 12);
2679 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2680 rdev->mc.gtt_end >> 12);
2682 /* VRAM after AGP */
2683 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2684 rdev->mc.gtt_start >> 12);
2685 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2686 rdev->mc.vram_end >> 12);
2689 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2690 rdev->mc.vram_start >> 12);
2691 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2692 rdev->mc.vram_end >> 12);
2694 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2695 /* llano/ontario only */
2696 if ((rdev->family == CHIP_PALM) ||
2697 (rdev->family == CHIP_SUMO) ||
2698 (rdev->family == CHIP_SUMO2)) {
2699 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2700 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2701 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2702 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2704 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2705 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2706 WREG32(MC_VM_FB_LOCATION, tmp);
2707 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2708 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2709 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2710 if (rdev->flags & RADEON_IS_AGP) {
2711 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2712 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2713 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2715 WREG32(MC_VM_AGP_BASE, 0);
2716 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2717 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2719 if (evergreen_mc_wait_for_idle(rdev)) {
2720 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2722 evergreen_mc_resume(rdev, &save);
2723 /* we need to own VRAM, so turn off the VGA renderer here
2724 * to stop it overwriting our objects */
2725 rv515_vga_render_disable(rdev);
2731 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2733 struct radeon_ring *ring = &rdev->ring[ib->ring];
2736 /* set to DX10/11 mode */
2737 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2738 radeon_ring_write(ring, 1);
2740 if (ring->rptr_save_reg) {
2741 next_rptr = ring->wptr + 3 + 4;
2742 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2743 radeon_ring_write(ring, ((ring->rptr_save_reg -
2744 PACKET3_SET_CONFIG_REG_START) >> 2));
2745 radeon_ring_write(ring, next_rptr);
2746 } else if (rdev->wb.enabled) {
2747 next_rptr = ring->wptr + 5 + 4;
2748 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2749 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2750 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2751 radeon_ring_write(ring, next_rptr);
2752 radeon_ring_write(ring, 0);
2755 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2756 radeon_ring_write(ring,
2760 (ib->gpu_addr & 0xFFFFFFFC));
2761 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2762 radeon_ring_write(ring, ib->length_dw);
2766 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2768 const __be32 *fw_data;
2771 if (!rdev->me_fw || !rdev->pfp_fw)
2779 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2781 fw_data = (const __be32 *)rdev->pfp_fw->data;
2782 WREG32(CP_PFP_UCODE_ADDR, 0);
2783 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2784 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2785 WREG32(CP_PFP_UCODE_ADDR, 0);
2787 fw_data = (const __be32 *)rdev->me_fw->data;
2788 WREG32(CP_ME_RAM_WADDR, 0);
2789 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2790 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2792 WREG32(CP_PFP_UCODE_ADDR, 0);
2793 WREG32(CP_ME_RAM_WADDR, 0);
2794 WREG32(CP_ME_RAM_RADDR, 0);
2798 static int evergreen_cp_start(struct radeon_device *rdev)
2800 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2804 r = radeon_ring_lock(rdev, ring, 7);
2806 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2809 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2810 radeon_ring_write(ring, 0x1);
2811 radeon_ring_write(ring, 0x0);
2812 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2813 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2814 radeon_ring_write(ring, 0);
2815 radeon_ring_write(ring, 0);
2816 radeon_ring_unlock_commit(rdev, ring);
2819 WREG32(CP_ME_CNTL, cp_me);
2821 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2823 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2827 /* setup clear context state */
2828 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2829 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2831 for (i = 0; i < evergreen_default_size; i++)
2832 radeon_ring_write(ring, evergreen_default_state[i]);
2834 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2835 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2837 /* set clear context state */
2838 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2839 radeon_ring_write(ring, 0);
2841 /* SQ_VTX_BASE_VTX_LOC */
2842 radeon_ring_write(ring, 0xc0026f00);
2843 radeon_ring_write(ring, 0x00000000);
2844 radeon_ring_write(ring, 0x00000000);
2845 radeon_ring_write(ring, 0x00000000);
2848 radeon_ring_write(ring, 0xc0036f00);
2849 radeon_ring_write(ring, 0x00000bc4);
2850 radeon_ring_write(ring, 0xffffffff);
2851 radeon_ring_write(ring, 0xffffffff);
2852 radeon_ring_write(ring, 0xffffffff);
2854 radeon_ring_write(ring, 0xc0026900);
2855 radeon_ring_write(ring, 0x00000316);
2856 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2857 radeon_ring_write(ring, 0x00000010); /* */
2859 radeon_ring_unlock_commit(rdev, ring);
2864 static int evergreen_cp_resume(struct radeon_device *rdev)
2866 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2871 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2872 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2878 RREG32(GRBM_SOFT_RESET);
2880 WREG32(GRBM_SOFT_RESET, 0);
2881 RREG32(GRBM_SOFT_RESET);
2883 /* Set ring buffer size */
2884 rb_bufsz = drm_order(ring->ring_size / 8);
2885 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2887 tmp |= BUF_SWAP_32BIT;
2889 WREG32(CP_RB_CNTL, tmp);
2890 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2891 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2893 /* Set the write pointer delay */
2894 WREG32(CP_RB_WPTR_DELAY, 0);
2896 /* Initialize the ring buffer's read and write pointers */
2897 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2898 WREG32(CP_RB_RPTR_WR, 0);
2900 WREG32(CP_RB_WPTR, ring->wptr);
2902 /* set the wb address whether it's enabled or not */
2903 WREG32(CP_RB_RPTR_ADDR,
2904 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2905 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2906 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2908 if (rdev->wb.enabled)
2909 WREG32(SCRATCH_UMSK, 0xff);
2911 tmp |= RB_NO_UPDATE;
2912 WREG32(SCRATCH_UMSK, 0);
2916 WREG32(CP_RB_CNTL, tmp);
2918 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2919 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2921 ring->rptr = RREG32(CP_RB_RPTR);
2923 evergreen_cp_start(rdev);
2925 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2927 ring->ready = false;
2936 static void evergreen_gpu_init(struct radeon_device *rdev)
2939 u32 mc_shared_chmap, mc_arb_ramcfg;
2943 u32 sq_lds_resource_mgmt;
2944 u32 sq_gpr_resource_mgmt_1;
2945 u32 sq_gpr_resource_mgmt_2;
2946 u32 sq_gpr_resource_mgmt_3;
2947 u32 sq_thread_resource_mgmt;
2948 u32 sq_thread_resource_mgmt_2;
2949 u32 sq_stack_resource_mgmt_1;
2950 u32 sq_stack_resource_mgmt_2;
2951 u32 sq_stack_resource_mgmt_3;
2952 u32 vgt_cache_invalidation;
2953 u32 hdp_host_path_cntl, tmp;
2954 u32 disabled_rb_mask;
2955 int i, j, num_shader_engines, ps_thread_count;
2957 switch (rdev->family) {
2960 rdev->config.evergreen.num_ses = 2;
2961 rdev->config.evergreen.max_pipes = 4;
2962 rdev->config.evergreen.max_tile_pipes = 8;
2963 rdev->config.evergreen.max_simds = 10;
2964 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2965 rdev->config.evergreen.max_gprs = 256;
2966 rdev->config.evergreen.max_threads = 248;
2967 rdev->config.evergreen.max_gs_threads = 32;
2968 rdev->config.evergreen.max_stack_entries = 512;
2969 rdev->config.evergreen.sx_num_of_sets = 4;
2970 rdev->config.evergreen.sx_max_export_size = 256;
2971 rdev->config.evergreen.sx_max_export_pos_size = 64;
2972 rdev->config.evergreen.sx_max_export_smx_size = 192;
2973 rdev->config.evergreen.max_hw_contexts = 8;
2974 rdev->config.evergreen.sq_num_cf_insts = 2;
2976 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2977 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2978 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2979 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
2982 rdev->config.evergreen.num_ses = 1;
2983 rdev->config.evergreen.max_pipes = 4;
2984 rdev->config.evergreen.max_tile_pipes = 4;
2985 rdev->config.evergreen.max_simds = 10;
2986 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2987 rdev->config.evergreen.max_gprs = 256;
2988 rdev->config.evergreen.max_threads = 248;
2989 rdev->config.evergreen.max_gs_threads = 32;
2990 rdev->config.evergreen.max_stack_entries = 512;
2991 rdev->config.evergreen.sx_num_of_sets = 4;
2992 rdev->config.evergreen.sx_max_export_size = 256;
2993 rdev->config.evergreen.sx_max_export_pos_size = 64;
2994 rdev->config.evergreen.sx_max_export_smx_size = 192;
2995 rdev->config.evergreen.max_hw_contexts = 8;
2996 rdev->config.evergreen.sq_num_cf_insts = 2;
2998 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2999 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3000 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3001 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3004 rdev->config.evergreen.num_ses = 1;
3005 rdev->config.evergreen.max_pipes = 4;
3006 rdev->config.evergreen.max_tile_pipes = 4;
3007 rdev->config.evergreen.max_simds = 5;
3008 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3009 rdev->config.evergreen.max_gprs = 256;
3010 rdev->config.evergreen.max_threads = 248;
3011 rdev->config.evergreen.max_gs_threads = 32;
3012 rdev->config.evergreen.max_stack_entries = 256;
3013 rdev->config.evergreen.sx_num_of_sets = 4;
3014 rdev->config.evergreen.sx_max_export_size = 256;
3015 rdev->config.evergreen.sx_max_export_pos_size = 64;
3016 rdev->config.evergreen.sx_max_export_smx_size = 192;
3017 rdev->config.evergreen.max_hw_contexts = 8;
3018 rdev->config.evergreen.sq_num_cf_insts = 2;
3020 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3021 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3022 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3023 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3027 rdev->config.evergreen.num_ses = 1;
3028 rdev->config.evergreen.max_pipes = 2;
3029 rdev->config.evergreen.max_tile_pipes = 2;
3030 rdev->config.evergreen.max_simds = 2;
3031 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3032 rdev->config.evergreen.max_gprs = 256;
3033 rdev->config.evergreen.max_threads = 192;
3034 rdev->config.evergreen.max_gs_threads = 16;
3035 rdev->config.evergreen.max_stack_entries = 256;
3036 rdev->config.evergreen.sx_num_of_sets = 4;
3037 rdev->config.evergreen.sx_max_export_size = 128;
3038 rdev->config.evergreen.sx_max_export_pos_size = 32;
3039 rdev->config.evergreen.sx_max_export_smx_size = 96;
3040 rdev->config.evergreen.max_hw_contexts = 4;
3041 rdev->config.evergreen.sq_num_cf_insts = 1;
3043 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3044 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3045 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3046 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3049 rdev->config.evergreen.num_ses = 1;
3050 rdev->config.evergreen.max_pipes = 2;
3051 rdev->config.evergreen.max_tile_pipes = 2;
3052 rdev->config.evergreen.max_simds = 2;
3053 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3054 rdev->config.evergreen.max_gprs = 256;
3055 rdev->config.evergreen.max_threads = 192;
3056 rdev->config.evergreen.max_gs_threads = 16;
3057 rdev->config.evergreen.max_stack_entries = 256;
3058 rdev->config.evergreen.sx_num_of_sets = 4;
3059 rdev->config.evergreen.sx_max_export_size = 128;
3060 rdev->config.evergreen.sx_max_export_pos_size = 32;
3061 rdev->config.evergreen.sx_max_export_smx_size = 96;
3062 rdev->config.evergreen.max_hw_contexts = 4;
3063 rdev->config.evergreen.sq_num_cf_insts = 1;
3065 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3066 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3067 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3068 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3071 rdev->config.evergreen.num_ses = 1;
3072 rdev->config.evergreen.max_pipes = 4;
3073 rdev->config.evergreen.max_tile_pipes = 4;
3074 if (rdev->pdev->device == 0x9648)
3075 rdev->config.evergreen.max_simds = 3;
3076 else if ((rdev->pdev->device == 0x9647) ||
3077 (rdev->pdev->device == 0x964a))
3078 rdev->config.evergreen.max_simds = 4;
3080 rdev->config.evergreen.max_simds = 5;
3081 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3082 rdev->config.evergreen.max_gprs = 256;
3083 rdev->config.evergreen.max_threads = 248;
3084 rdev->config.evergreen.max_gs_threads = 32;
3085 rdev->config.evergreen.max_stack_entries = 256;
3086 rdev->config.evergreen.sx_num_of_sets = 4;
3087 rdev->config.evergreen.sx_max_export_size = 256;
3088 rdev->config.evergreen.sx_max_export_pos_size = 64;
3089 rdev->config.evergreen.sx_max_export_smx_size = 192;
3090 rdev->config.evergreen.max_hw_contexts = 8;
3091 rdev->config.evergreen.sq_num_cf_insts = 2;
3093 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3094 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3095 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3096 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3099 rdev->config.evergreen.num_ses = 1;
3100 rdev->config.evergreen.max_pipes = 4;
3101 rdev->config.evergreen.max_tile_pipes = 4;
3102 rdev->config.evergreen.max_simds = 2;
3103 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3104 rdev->config.evergreen.max_gprs = 256;
3105 rdev->config.evergreen.max_threads = 248;
3106 rdev->config.evergreen.max_gs_threads = 32;
3107 rdev->config.evergreen.max_stack_entries = 512;
3108 rdev->config.evergreen.sx_num_of_sets = 4;
3109 rdev->config.evergreen.sx_max_export_size = 256;
3110 rdev->config.evergreen.sx_max_export_pos_size = 64;
3111 rdev->config.evergreen.sx_max_export_smx_size = 192;
3112 rdev->config.evergreen.max_hw_contexts = 8;
3113 rdev->config.evergreen.sq_num_cf_insts = 2;
3115 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3116 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3117 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3118 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3121 rdev->config.evergreen.num_ses = 2;
3122 rdev->config.evergreen.max_pipes = 4;
3123 rdev->config.evergreen.max_tile_pipes = 8;
3124 rdev->config.evergreen.max_simds = 7;
3125 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3126 rdev->config.evergreen.max_gprs = 256;
3127 rdev->config.evergreen.max_threads = 248;
3128 rdev->config.evergreen.max_gs_threads = 32;
3129 rdev->config.evergreen.max_stack_entries = 512;
3130 rdev->config.evergreen.sx_num_of_sets = 4;
3131 rdev->config.evergreen.sx_max_export_size = 256;
3132 rdev->config.evergreen.sx_max_export_pos_size = 64;
3133 rdev->config.evergreen.sx_max_export_smx_size = 192;
3134 rdev->config.evergreen.max_hw_contexts = 8;
3135 rdev->config.evergreen.sq_num_cf_insts = 2;
3137 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3138 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3139 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3140 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3143 rdev->config.evergreen.num_ses = 1;
3144 rdev->config.evergreen.max_pipes = 4;
3145 rdev->config.evergreen.max_tile_pipes = 4;
3146 rdev->config.evergreen.max_simds = 6;
3147 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3148 rdev->config.evergreen.max_gprs = 256;
3149 rdev->config.evergreen.max_threads = 248;
3150 rdev->config.evergreen.max_gs_threads = 32;
3151 rdev->config.evergreen.max_stack_entries = 256;
3152 rdev->config.evergreen.sx_num_of_sets = 4;
3153 rdev->config.evergreen.sx_max_export_size = 256;
3154 rdev->config.evergreen.sx_max_export_pos_size = 64;
3155 rdev->config.evergreen.sx_max_export_smx_size = 192;
3156 rdev->config.evergreen.max_hw_contexts = 8;
3157 rdev->config.evergreen.sq_num_cf_insts = 2;
3159 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3160 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3161 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3162 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3165 rdev->config.evergreen.num_ses = 1;
3166 rdev->config.evergreen.max_pipes = 2;
3167 rdev->config.evergreen.max_tile_pipes = 2;
3168 rdev->config.evergreen.max_simds = 2;
3169 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3170 rdev->config.evergreen.max_gprs = 256;
3171 rdev->config.evergreen.max_threads = 192;
3172 rdev->config.evergreen.max_gs_threads = 16;
3173 rdev->config.evergreen.max_stack_entries = 256;
3174 rdev->config.evergreen.sx_num_of_sets = 4;
3175 rdev->config.evergreen.sx_max_export_size = 128;
3176 rdev->config.evergreen.sx_max_export_pos_size = 32;
3177 rdev->config.evergreen.sx_max_export_smx_size = 96;
3178 rdev->config.evergreen.max_hw_contexts = 4;
3179 rdev->config.evergreen.sq_num_cf_insts = 1;
3181 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3182 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3183 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3184 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3188 /* Initialize HDP */
3189 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3190 WREG32((0x2c14 + j), 0x00000000);
3191 WREG32((0x2c18 + j), 0x00000000);
3192 WREG32((0x2c1c + j), 0x00000000);
3193 WREG32((0x2c20 + j), 0x00000000);
3194 WREG32((0x2c24 + j), 0x00000000);
3197 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3199 evergreen_fix_pci_max_read_req_size(rdev);
3201 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3202 if ((rdev->family == CHIP_PALM) ||
3203 (rdev->family == CHIP_SUMO) ||
3204 (rdev->family == CHIP_SUMO2))
3205 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3207 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3209 /* setup tiling info dword. gb_addr_config is not adequate since it does
3210 * not have bank info, so create a custom tiling dword.
3211 * bits 3:0 num_pipes
3212 * bits 7:4 num_banks
3213 * bits 11:8 group_size
3214 * bits 15:12 row_size
3216 rdev->config.evergreen.tile_config = 0;
3217 switch (rdev->config.evergreen.max_tile_pipes) {
3220 rdev->config.evergreen.tile_config |= (0 << 0);
3223 rdev->config.evergreen.tile_config |= (1 << 0);
3226 rdev->config.evergreen.tile_config |= (2 << 0);
3229 rdev->config.evergreen.tile_config |= (3 << 0);
3232 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3233 if (rdev->flags & RADEON_IS_IGP)
3234 rdev->config.evergreen.tile_config |= 1 << 4;
3236 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3237 case 0: /* four banks */
3238 rdev->config.evergreen.tile_config |= 0 << 4;
3240 case 1: /* eight banks */
3241 rdev->config.evergreen.tile_config |= 1 << 4;
3243 case 2: /* sixteen banks */
3245 rdev->config.evergreen.tile_config |= 2 << 4;
3249 rdev->config.evergreen.tile_config |= 0 << 8;
3250 rdev->config.evergreen.tile_config |=
3251 ((gb_addr_config & 0x30000000) >> 28) << 12;
3253 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3255 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3259 efuse_straps_4 = RREG32_RCU(0x204);
3260 efuse_straps_3 = RREG32_RCU(0x203);
3261 tmp = (((efuse_straps_4 & 0xf) << 4) |
3262 ((efuse_straps_3 & 0xf0000000) >> 28));
3265 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3266 u32 rb_disable_bitmap;
3268 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3269 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3270 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3272 tmp |= rb_disable_bitmap;
3275 /* enabled rb are just the one not disabled :) */
3276 disabled_rb_mask = tmp;
3278 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3280 /* if all the backends are disabled, fix it up here */
3281 if ((disabled_rb_mask & tmp) == tmp) {
3282 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3283 disabled_rb_mask &= ~(1 << i);
3286 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3287 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3289 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3290 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3291 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3292 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3293 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3294 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3295 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3297 if ((rdev->config.evergreen.max_backends == 1) &&
3298 (rdev->flags & RADEON_IS_IGP)) {
3299 if ((disabled_rb_mask & 3) == 1) {
3300 /* RB0 disabled, RB1 enabled */
3303 /* RB1 disabled, RB0 enabled */
3307 tmp = gb_addr_config & NUM_PIPES_MASK;
3308 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3309 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3311 WREG32(GB_BACKEND_MAP, tmp);
3313 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3314 WREG32(CGTS_TCC_DISABLE, 0);
3315 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3316 WREG32(CGTS_USER_TCC_DISABLE, 0);
3318 /* set HW defaults for 3D engine */
3319 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3320 ROQ_IB2_START(0x2b)));
3322 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3324 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3329 sx_debug_1 = RREG32(SX_DEBUG_1);
3330 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3331 WREG32(SX_DEBUG_1, sx_debug_1);
3334 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3335 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3336 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3337 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3339 if (rdev->family <= CHIP_SUMO2)
3340 WREG32(SMX_SAR_CTL0, 0x00010000);
3342 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3343 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3344 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3346 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3347 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3348 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3350 WREG32(VGT_NUM_INSTANCES, 1);
3351 WREG32(SPI_CONFIG_CNTL, 0);
3352 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3353 WREG32(CP_PERFMON_CNTL, 0);
3355 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3356 FETCH_FIFO_HIWATER(0x4) |
3357 DONE_FIFO_HIWATER(0xe0) |
3358 ALU_UPDATE_FIFO_HIWATER(0x8)));
3360 sq_config = RREG32(SQ_CONFIG);
3361 sq_config &= ~(PS_PRIO(3) |
3365 sq_config |= (VC_ENABLE |
3372 switch (rdev->family) {
3378 /* no vertex cache */
3379 sq_config &= ~VC_ENABLE;
3385 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3387 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3388 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3389 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3390 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3391 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3392 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3393 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3395 switch (rdev->family) {
3400 ps_thread_count = 96;
3403 ps_thread_count = 128;
3407 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3408 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3409 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3410 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3411 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3412 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3414 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3415 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3416 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3417 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3418 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3419 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3421 WREG32(SQ_CONFIG, sq_config);
3422 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3423 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3424 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3425 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3426 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3427 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3428 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3429 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3430 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3431 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3433 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3434 FORCE_EOV_MAX_REZ_CNT(255)));
3436 switch (rdev->family) {
3442 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3445 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3448 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3449 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3451 WREG32(VGT_GS_VERTEX_REUSE, 16);
3452 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3453 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3455 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3456 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3458 WREG32(CB_PERF_CTR0_SEL_0, 0);
3459 WREG32(CB_PERF_CTR0_SEL_1, 0);
3460 WREG32(CB_PERF_CTR1_SEL_0, 0);
3461 WREG32(CB_PERF_CTR1_SEL_1, 0);
3462 WREG32(CB_PERF_CTR2_SEL_0, 0);
3463 WREG32(CB_PERF_CTR2_SEL_1, 0);
3464 WREG32(CB_PERF_CTR3_SEL_0, 0);
3465 WREG32(CB_PERF_CTR3_SEL_1, 0);
3467 /* clear render buffer base addresses */
3468 WREG32(CB_COLOR0_BASE, 0);
3469 WREG32(CB_COLOR1_BASE, 0);
3470 WREG32(CB_COLOR2_BASE, 0);
3471 WREG32(CB_COLOR3_BASE, 0);
3472 WREG32(CB_COLOR4_BASE, 0);
3473 WREG32(CB_COLOR5_BASE, 0);
3474 WREG32(CB_COLOR6_BASE, 0);
3475 WREG32(CB_COLOR7_BASE, 0);
3476 WREG32(CB_COLOR8_BASE, 0);
3477 WREG32(CB_COLOR9_BASE, 0);
3478 WREG32(CB_COLOR10_BASE, 0);
3479 WREG32(CB_COLOR11_BASE, 0);
3481 /* set the shader const cache sizes to 0 */
3482 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3484 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3487 tmp = RREG32(HDP_MISC_CNTL);
3488 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3489 WREG32(HDP_MISC_CNTL, tmp);
3491 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3492 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3494 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3500 int evergreen_mc_init(struct radeon_device *rdev)
3503 int chansize, numchan;
3505 /* Get VRAM informations */
3506 rdev->mc.vram_is_ddr = true;
3507 if ((rdev->family == CHIP_PALM) ||
3508 (rdev->family == CHIP_SUMO) ||
3509 (rdev->family == CHIP_SUMO2))
3510 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3512 tmp = RREG32(MC_ARB_RAMCFG);
3513 if (tmp & CHANSIZE_OVERRIDE) {
3515 } else if (tmp & CHANSIZE_MASK) {
3520 tmp = RREG32(MC_SHARED_CHMAP);
3521 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3536 rdev->mc.vram_width = numchan * chansize;
3537 /* Could aper size report 0 ? */
3538 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3539 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3540 /* Setup GPU memory space */
3541 if ((rdev->family == CHIP_PALM) ||
3542 (rdev->family == CHIP_SUMO) ||
3543 (rdev->family == CHIP_SUMO2)) {
3544 /* size in bytes on fusion */
3545 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3546 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3548 /* size in MB on evergreen/cayman/tn */
3549 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3550 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3552 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3553 r700_vram_gtt_location(rdev, &rdev->mc);
3554 radeon_update_bandwidth_info(rdev);
3559 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3561 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3562 RREG32(GRBM_STATUS));
3563 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3564 RREG32(GRBM_STATUS_SE0));
3565 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3566 RREG32(GRBM_STATUS_SE1));
3567 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3568 RREG32(SRBM_STATUS));
3569 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3570 RREG32(SRBM_STATUS2));
3571 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3572 RREG32(CP_STALLED_STAT1));
3573 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3574 RREG32(CP_STALLED_STAT2));
3575 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3576 RREG32(CP_BUSY_STAT));
3577 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3579 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3580 RREG32(DMA_STATUS_REG));
3581 if (rdev->family >= CHIP_CAYMAN) {
3582 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3583 RREG32(DMA_STATUS_REG + 0x800));
3587 bool evergreen_is_display_hung(struct radeon_device *rdev)
3593 for (i = 0; i < rdev->num_crtc; i++) {
3594 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3595 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3596 crtc_hung |= (1 << i);
3600 for (j = 0; j < 10; j++) {
3601 for (i = 0; i < rdev->num_crtc; i++) {
3602 if (crtc_hung & (1 << i)) {
3603 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3604 if (tmp != crtc_status[i])
3605 crtc_hung &= ~(1 << i);
3616 static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3622 tmp = RREG32(GRBM_STATUS);
3623 if (tmp & (PA_BUSY | SC_BUSY |
3625 TA_BUSY | VGT_BUSY |
3627 SPI_BUSY | VGT_BUSY_NO_DMA))
3628 reset_mask |= RADEON_RESET_GFX;
3630 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3631 CP_BUSY | CP_COHERENCY_BUSY))
3632 reset_mask |= RADEON_RESET_CP;
3634 if (tmp & GRBM_EE_BUSY)
3635 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3637 /* DMA_STATUS_REG */
3638 tmp = RREG32(DMA_STATUS_REG);
3639 if (!(tmp & DMA_IDLE))
3640 reset_mask |= RADEON_RESET_DMA;
3643 tmp = RREG32(SRBM_STATUS2);
3645 reset_mask |= RADEON_RESET_DMA;
3648 tmp = RREG32(SRBM_STATUS);
3649 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3650 reset_mask |= RADEON_RESET_RLC;
3653 reset_mask |= RADEON_RESET_IH;
3656 reset_mask |= RADEON_RESET_SEM;
3658 if (tmp & GRBM_RQ_PENDING)
3659 reset_mask |= RADEON_RESET_GRBM;
3662 reset_mask |= RADEON_RESET_VMC;
3664 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3665 MCC_BUSY | MCD_BUSY))
3666 reset_mask |= RADEON_RESET_MC;
3668 if (evergreen_is_display_hung(rdev))
3669 reset_mask |= RADEON_RESET_DISPLAY;
3672 tmp = RREG32(VM_L2_STATUS);
3674 reset_mask |= RADEON_RESET_VMC;
3676 /* Skip MC reset as it's mostly likely not hung, just busy */
3677 if (reset_mask & RADEON_RESET_MC) {
3678 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3679 reset_mask &= ~RADEON_RESET_MC;
3685 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3687 struct evergreen_mc_save save;
3688 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3691 if (reset_mask == 0)
3694 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3696 evergreen_print_gpu_status_regs(rdev);
3698 /* Disable CP parsing/prefetching */
3699 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3701 if (reset_mask & RADEON_RESET_DMA) {
3703 tmp = RREG32(DMA_RB_CNTL);
3704 tmp &= ~DMA_RB_ENABLE;
3705 WREG32(DMA_RB_CNTL, tmp);
3710 evergreen_mc_stop(rdev, &save);
3711 if (evergreen_mc_wait_for_idle(rdev)) {
3712 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3715 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3716 grbm_soft_reset |= SOFT_RESET_DB |
3729 if (reset_mask & RADEON_RESET_CP) {
3730 grbm_soft_reset |= SOFT_RESET_CP |
3733 srbm_soft_reset |= SOFT_RESET_GRBM;
3736 if (reset_mask & RADEON_RESET_DMA)
3737 srbm_soft_reset |= SOFT_RESET_DMA;
3739 if (reset_mask & RADEON_RESET_DISPLAY)
3740 srbm_soft_reset |= SOFT_RESET_DC;
3742 if (reset_mask & RADEON_RESET_RLC)
3743 srbm_soft_reset |= SOFT_RESET_RLC;
3745 if (reset_mask & RADEON_RESET_SEM)
3746 srbm_soft_reset |= SOFT_RESET_SEM;
3748 if (reset_mask & RADEON_RESET_IH)
3749 srbm_soft_reset |= SOFT_RESET_IH;
3751 if (reset_mask & RADEON_RESET_GRBM)
3752 srbm_soft_reset |= SOFT_RESET_GRBM;
3754 if (reset_mask & RADEON_RESET_VMC)
3755 srbm_soft_reset |= SOFT_RESET_VMC;
3757 if (!(rdev->flags & RADEON_IS_IGP)) {
3758 if (reset_mask & RADEON_RESET_MC)
3759 srbm_soft_reset |= SOFT_RESET_MC;
3762 if (grbm_soft_reset) {
3763 tmp = RREG32(GRBM_SOFT_RESET);
3764 tmp |= grbm_soft_reset;
3765 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3766 WREG32(GRBM_SOFT_RESET, tmp);
3767 tmp = RREG32(GRBM_SOFT_RESET);
3771 tmp &= ~grbm_soft_reset;
3772 WREG32(GRBM_SOFT_RESET, tmp);
3773 tmp = RREG32(GRBM_SOFT_RESET);
3776 if (srbm_soft_reset) {
3777 tmp = RREG32(SRBM_SOFT_RESET);
3778 tmp |= srbm_soft_reset;
3779 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3780 WREG32(SRBM_SOFT_RESET, tmp);
3781 tmp = RREG32(SRBM_SOFT_RESET);
3785 tmp &= ~srbm_soft_reset;
3786 WREG32(SRBM_SOFT_RESET, tmp);
3787 tmp = RREG32(SRBM_SOFT_RESET);
3790 /* Wait a little for things to settle down */
3793 evergreen_mc_resume(rdev, &save);
3796 evergreen_print_gpu_status_regs(rdev);
3799 int evergreen_asic_reset(struct radeon_device *rdev)
3803 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3806 r600_set_bios_scratch_engine_hung(rdev, true);
3808 evergreen_gpu_soft_reset(rdev, reset_mask);
3810 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3813 r600_set_bios_scratch_engine_hung(rdev, false);
3819 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3821 * @rdev: radeon_device pointer
3822 * @ring: radeon_ring structure holding ring information
3824 * Check if the GFX engine is locked up.
3825 * Returns true if the engine appears to be locked up, false if not.
3827 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3829 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3831 if (!(reset_mask & (RADEON_RESET_GFX |
3832 RADEON_RESET_COMPUTE |
3833 RADEON_RESET_CP))) {
3834 radeon_ring_lockup_update(ring);
3837 /* force CP activities */
3838 radeon_ring_force_activity(rdev, ring);
3839 return radeon_ring_test_lockup(rdev, ring);
3843 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3845 * @rdev: radeon_device pointer
3846 * @ring: radeon_ring structure holding ring information
3848 * Check if the async DMA engine is locked up.
3849 * Returns true if the engine appears to be locked up, false if not.
3851 bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3853 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3855 if (!(reset_mask & RADEON_RESET_DMA)) {
3856 radeon_ring_lockup_update(ring);
3859 /* force ring activities */
3860 radeon_ring_force_activity(rdev, ring);
3861 return radeon_ring_test_lockup(rdev, ring);
3867 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3868 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
3870 void sumo_rlc_fini(struct radeon_device *rdev)
3874 /* save restore block */
3875 if (rdev->rlc.save_restore_obj) {
3876 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3877 if (unlikely(r != 0))
3878 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3879 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3880 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3882 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3883 rdev->rlc.save_restore_obj = NULL;
3886 /* clear state block */
3887 if (rdev->rlc.clear_state_obj) {
3888 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3889 if (unlikely(r != 0))
3890 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3891 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3892 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3894 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3895 rdev->rlc.clear_state_obj = NULL;
3899 int sumo_rlc_init(struct radeon_device *rdev)
3902 volatile u32 *dst_ptr;
3903 u32 dws, data, i, j, k, reg_num;
3904 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
3905 u64 reg_list_mc_addr;
3906 struct cs_section_def *cs_data;
3909 src_ptr = rdev->rlc.reg_list;
3910 dws = rdev->rlc.reg_list_size;
3911 cs_data = rdev->rlc.cs_data;
3913 /* save restore block */
3914 if (rdev->rlc.save_restore_obj == NULL) {
3915 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3916 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3918 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3923 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3924 if (unlikely(r != 0)) {
3925 sumo_rlc_fini(rdev);
3928 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3929 &rdev->rlc.save_restore_gpu_addr);
3931 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3932 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3933 sumo_rlc_fini(rdev);
3936 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3938 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3939 sumo_rlc_fini(rdev);
3942 /* write the sr buffer */
3943 dst_ptr = rdev->rlc.sr_ptr;
3945 * dw0: (reg2 << 16) | reg1
3946 * dw1: reg1 save space
3947 * dw2: reg2 save space
3949 for (i = 0; i < dws; i++) {
3950 data = src_ptr[i] >> 2;
3953 data |= (src_ptr[i] >> 2) << 16;
3954 j = (((i - 1) * 3) / 2);
3958 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
3960 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3961 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3963 /* clear state block */
3966 for (i = 0; cs_data[i].section != NULL; i++) {
3967 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3969 dws += cs_data[i].section[j].reg_count;
3972 reg_list_blk_index = (3 * reg_list_num + 2);
3973 dws += reg_list_blk_index;
3975 if (rdev->rlc.clear_state_obj == NULL) {
3976 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3977 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
3979 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
3980 sumo_rlc_fini(rdev);
3984 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3985 if (unlikely(r != 0)) {
3986 sumo_rlc_fini(rdev);
3989 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
3990 &rdev->rlc.clear_state_gpu_addr);
3993 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3994 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
3995 sumo_rlc_fini(rdev);
3998 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4000 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4001 sumo_rlc_fini(rdev);
4004 /* set up the cs buffer */
4005 dst_ptr = rdev->rlc.cs_ptr;
4006 reg_list_hdr_blk_index = 0;
4007 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4008 data = upper_32_bits(reg_list_mc_addr);
4009 dst_ptr[reg_list_hdr_blk_index] = data;
4010 reg_list_hdr_blk_index++;
4011 for (i = 0; cs_data[i].section != NULL; i++) {
4012 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4013 reg_num = cs_data[i].section[j].reg_count;
4014 data = reg_list_mc_addr & 0xffffffff;
4015 dst_ptr[reg_list_hdr_blk_index] = data;
4016 reg_list_hdr_blk_index++;
4018 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4019 dst_ptr[reg_list_hdr_blk_index] = data;
4020 reg_list_hdr_blk_index++;
4022 data = 0x08000000 | (reg_num * 4);
4023 dst_ptr[reg_list_hdr_blk_index] = data;
4024 reg_list_hdr_blk_index++;
4026 for (k = 0; k < reg_num; k++) {
4027 data = cs_data[i].section[j].extent[k];
4028 dst_ptr[reg_list_blk_index + k] = data;
4030 reg_list_mc_addr += reg_num * 4;
4031 reg_list_blk_index += reg_num;
4034 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4036 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4037 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4042 static void evergreen_rlc_start(struct radeon_device *rdev)
4044 u32 mask = RLC_ENABLE;
4046 if (rdev->flags & RADEON_IS_IGP) {
4047 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4050 WREG32(RLC_CNTL, mask);
4053 int evergreen_rlc_resume(struct radeon_device *rdev)
4056 const __be32 *fw_data;
4061 r600_rlc_stop(rdev);
4063 WREG32(RLC_HB_CNTL, 0);
4065 if (rdev->flags & RADEON_IS_IGP) {
4066 if (rdev->family == CHIP_ARUBA) {
4067 u32 always_on_bitmap =
4068 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4069 /* find out the number of active simds */
4070 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4071 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4072 tmp = hweight32(~tmp);
4073 if (tmp == rdev->config.cayman.max_simds_per_se) {
4074 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4075 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4076 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4077 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4078 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4081 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4082 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4084 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4085 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4087 WREG32(RLC_HB_BASE, 0);
4088 WREG32(RLC_HB_RPTR, 0);
4089 WREG32(RLC_HB_WPTR, 0);
4090 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4091 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4093 WREG32(RLC_MC_CNTL, 0);
4094 WREG32(RLC_UCODE_CNTL, 0);
4096 fw_data = (const __be32 *)rdev->rlc_fw->data;
4097 if (rdev->family >= CHIP_ARUBA) {
4098 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4099 WREG32(RLC_UCODE_ADDR, i);
4100 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4102 } else if (rdev->family >= CHIP_CAYMAN) {
4103 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4104 WREG32(RLC_UCODE_ADDR, i);
4105 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4108 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4109 WREG32(RLC_UCODE_ADDR, i);
4110 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4113 WREG32(RLC_UCODE_ADDR, 0);
4115 evergreen_rlc_start(rdev);
4122 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4124 if (crtc >= rdev->num_crtc)
4127 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4130 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4134 if (rdev->family >= CHIP_CAYMAN) {
4135 cayman_cp_int_cntl_setup(rdev, 0,
4136 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4137 cayman_cp_int_cntl_setup(rdev, 1, 0);
4138 cayman_cp_int_cntl_setup(rdev, 2, 0);
4139 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4140 WREG32(CAYMAN_DMA1_CNTL, tmp);
4142 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4143 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4144 WREG32(DMA_CNTL, tmp);
4145 WREG32(GRBM_INT_CNTL, 0);
4146 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4147 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4148 if (rdev->num_crtc >= 4) {
4149 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4150 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4152 if (rdev->num_crtc >= 6) {
4153 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4154 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4157 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4158 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4159 if (rdev->num_crtc >= 4) {
4160 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4161 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4163 if (rdev->num_crtc >= 6) {
4164 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4165 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4168 /* only one DAC on DCE6 */
4169 if (!ASIC_IS_DCE6(rdev))
4170 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4171 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4173 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4174 WREG32(DC_HPD1_INT_CONTROL, tmp);
4175 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4176 WREG32(DC_HPD2_INT_CONTROL, tmp);
4177 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4178 WREG32(DC_HPD3_INT_CONTROL, tmp);
4179 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4180 WREG32(DC_HPD4_INT_CONTROL, tmp);
4181 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4182 WREG32(DC_HPD5_INT_CONTROL, tmp);
4183 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4184 WREG32(DC_HPD6_INT_CONTROL, tmp);
4188 int evergreen_irq_set(struct radeon_device *rdev)
4190 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4191 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4192 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4193 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4194 u32 grbm_int_cntl = 0;
4195 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
4196 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4197 u32 dma_cntl, dma_cntl1 = 0;
4198 u32 thermal_int = 0;
4200 if (!rdev->irq.installed) {
4201 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4204 /* don't enable anything if the ih is disabled */
4205 if (!rdev->ih.enabled) {
4206 r600_disable_interrupts(rdev);
4207 /* force the active interrupt state to all disabled */
4208 evergreen_disable_interrupt_state(rdev);
4212 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4213 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4214 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4215 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4216 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4217 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4218 if (rdev->family == CHIP_ARUBA)
4219 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4220 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4222 thermal_int = RREG32(CG_THERMAL_INT) &
4223 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4225 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4226 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4227 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4228 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4229 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4230 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4232 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4234 if (rdev->family >= CHIP_CAYMAN) {
4235 /* enable CP interrupts on all rings */
4236 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4237 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4238 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4240 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4241 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4242 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4244 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4245 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4246 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4249 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4250 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4251 cp_int_cntl |= RB_INT_ENABLE;
4252 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4256 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4257 DRM_DEBUG("r600_irq_set: sw int dma\n");
4258 dma_cntl |= TRAP_ENABLE;
4261 if (rdev->family >= CHIP_CAYMAN) {
4262 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4263 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4264 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4265 dma_cntl1 |= TRAP_ENABLE;
4269 if (rdev->irq.dpm_thermal) {
4270 DRM_DEBUG("dpm thermal\n");
4271 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4274 if (rdev->irq.crtc_vblank_int[0] ||
4275 atomic_read(&rdev->irq.pflip[0])) {
4276 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4277 crtc1 |= VBLANK_INT_MASK;
4279 if (rdev->irq.crtc_vblank_int[1] ||
4280 atomic_read(&rdev->irq.pflip[1])) {
4281 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4282 crtc2 |= VBLANK_INT_MASK;
4284 if (rdev->irq.crtc_vblank_int[2] ||
4285 atomic_read(&rdev->irq.pflip[2])) {
4286 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4287 crtc3 |= VBLANK_INT_MASK;
4289 if (rdev->irq.crtc_vblank_int[3] ||
4290 atomic_read(&rdev->irq.pflip[3])) {
4291 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4292 crtc4 |= VBLANK_INT_MASK;
4294 if (rdev->irq.crtc_vblank_int[4] ||
4295 atomic_read(&rdev->irq.pflip[4])) {
4296 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4297 crtc5 |= VBLANK_INT_MASK;
4299 if (rdev->irq.crtc_vblank_int[5] ||
4300 atomic_read(&rdev->irq.pflip[5])) {
4301 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4302 crtc6 |= VBLANK_INT_MASK;
4304 if (rdev->irq.hpd[0]) {
4305 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4306 hpd1 |= DC_HPDx_INT_EN;
4308 if (rdev->irq.hpd[1]) {
4309 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4310 hpd2 |= DC_HPDx_INT_EN;
4312 if (rdev->irq.hpd[2]) {
4313 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4314 hpd3 |= DC_HPDx_INT_EN;
4316 if (rdev->irq.hpd[3]) {
4317 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4318 hpd4 |= DC_HPDx_INT_EN;
4320 if (rdev->irq.hpd[4]) {
4321 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4322 hpd5 |= DC_HPDx_INT_EN;
4324 if (rdev->irq.hpd[5]) {
4325 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4326 hpd6 |= DC_HPDx_INT_EN;
4328 if (rdev->irq.afmt[0]) {
4329 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4330 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4332 if (rdev->irq.afmt[1]) {
4333 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4334 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4336 if (rdev->irq.afmt[2]) {
4337 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4338 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4340 if (rdev->irq.afmt[3]) {
4341 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4342 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4344 if (rdev->irq.afmt[4]) {
4345 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4346 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4348 if (rdev->irq.afmt[5]) {
4349 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4350 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4353 if (rdev->family >= CHIP_CAYMAN) {
4354 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4355 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4356 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4358 WREG32(CP_INT_CNTL, cp_int_cntl);
4360 WREG32(DMA_CNTL, dma_cntl);
4362 if (rdev->family >= CHIP_CAYMAN)
4363 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4365 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4367 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4368 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4369 if (rdev->num_crtc >= 4) {
4370 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4371 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4373 if (rdev->num_crtc >= 6) {
4374 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4375 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4378 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4379 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4380 if (rdev->num_crtc >= 4) {
4381 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4382 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4384 if (rdev->num_crtc >= 6) {
4385 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4386 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4389 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4390 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4391 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4392 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4393 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4394 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4395 if (rdev->family == CHIP_ARUBA)
4396 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4398 WREG32(CG_THERMAL_INT, thermal_int);
4400 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4401 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4402 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4403 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4404 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4405 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4410 static void evergreen_irq_ack(struct radeon_device *rdev)
4414 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4415 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4416 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4417 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4418 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4419 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4420 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4421 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4422 if (rdev->num_crtc >= 4) {
4423 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4424 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4426 if (rdev->num_crtc >= 6) {
4427 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4428 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4431 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4432 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4433 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4434 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4435 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4436 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4438 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4439 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4440 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4441 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4442 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4443 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4444 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4445 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4446 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4447 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4448 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4449 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4451 if (rdev->num_crtc >= 4) {
4452 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4453 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4454 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4455 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4456 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4457 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4458 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4459 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4460 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4461 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4462 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4463 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4466 if (rdev->num_crtc >= 6) {
4467 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4468 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4469 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4470 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4471 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4472 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4473 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4474 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4475 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4476 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4477 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4478 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4481 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4482 tmp = RREG32(DC_HPD1_INT_CONTROL);
4483 tmp |= DC_HPDx_INT_ACK;
4484 WREG32(DC_HPD1_INT_CONTROL, tmp);
4486 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4487 tmp = RREG32(DC_HPD2_INT_CONTROL);
4488 tmp |= DC_HPDx_INT_ACK;
4489 WREG32(DC_HPD2_INT_CONTROL, tmp);
4491 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4492 tmp = RREG32(DC_HPD3_INT_CONTROL);
4493 tmp |= DC_HPDx_INT_ACK;
4494 WREG32(DC_HPD3_INT_CONTROL, tmp);
4496 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4497 tmp = RREG32(DC_HPD4_INT_CONTROL);
4498 tmp |= DC_HPDx_INT_ACK;
4499 WREG32(DC_HPD4_INT_CONTROL, tmp);
4501 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4502 tmp = RREG32(DC_HPD5_INT_CONTROL);
4503 tmp |= DC_HPDx_INT_ACK;
4504 WREG32(DC_HPD5_INT_CONTROL, tmp);
4506 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4507 tmp = RREG32(DC_HPD5_INT_CONTROL);
4508 tmp |= DC_HPDx_INT_ACK;
4509 WREG32(DC_HPD6_INT_CONTROL, tmp);
4511 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4512 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4513 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4514 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4516 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4517 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4518 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4519 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4521 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4522 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4523 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4524 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4526 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4527 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4528 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4529 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4531 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4532 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4533 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4534 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4536 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4537 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4538 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4539 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4543 static void evergreen_irq_disable(struct radeon_device *rdev)
4545 r600_disable_interrupts(rdev);
4546 /* Wait and acknowledge irq */
4548 evergreen_irq_ack(rdev);
4549 evergreen_disable_interrupt_state(rdev);
4552 void evergreen_irq_suspend(struct radeon_device *rdev)
4554 evergreen_irq_disable(rdev);
4555 r600_rlc_stop(rdev);
4558 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4562 if (rdev->wb.enabled)
4563 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4565 wptr = RREG32(IH_RB_WPTR);
4567 if (wptr & RB_OVERFLOW) {
4568 /* When a ring buffer overflow happen start parsing interrupt
4569 * from the last not overwritten vector (wptr + 16). Hopefully
4570 * this should allow us to catchup.
4572 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4573 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4574 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4575 tmp = RREG32(IH_RB_CNTL);
4576 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4577 WREG32(IH_RB_CNTL, tmp);
4579 return (wptr & rdev->ih.ptr_mask);
4582 int evergreen_irq_process(struct radeon_device *rdev)
4586 u32 src_id, src_data;
4588 bool queue_hotplug = false;
4589 bool queue_hdmi = false;
4590 bool queue_thermal = false;
4593 if (!rdev->ih.enabled || rdev->shutdown)
4596 wptr = evergreen_get_ih_wptr(rdev);
4599 /* is somebody else already processing irqs? */
4600 if (atomic_xchg(&rdev->ih.lock, 1))
4603 rptr = rdev->ih.rptr;
4604 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4606 /* Order reading of wptr vs. reading of IH ring data */
4609 /* display interrupts */
4610 evergreen_irq_ack(rdev);
4612 while (rptr != wptr) {
4613 /* wptr/rptr are in bytes! */
4614 ring_index = rptr / 4;
4615 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4616 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4619 case 1: /* D1 vblank/vline */
4621 case 0: /* D1 vblank */
4622 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4623 if (rdev->irq.crtc_vblank_int[0]) {
4624 drm_handle_vblank(rdev->ddev, 0);
4625 rdev->pm.vblank_sync = true;
4626 wake_up(&rdev->irq.vblank_queue);
4628 if (atomic_read(&rdev->irq.pflip[0]))
4629 radeon_crtc_handle_flip(rdev, 0);
4630 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4631 DRM_DEBUG("IH: D1 vblank\n");
4634 case 1: /* D1 vline */
4635 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4636 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4637 DRM_DEBUG("IH: D1 vline\n");
4641 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4645 case 2: /* D2 vblank/vline */
4647 case 0: /* D2 vblank */
4648 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4649 if (rdev->irq.crtc_vblank_int[1]) {
4650 drm_handle_vblank(rdev->ddev, 1);
4651 rdev->pm.vblank_sync = true;
4652 wake_up(&rdev->irq.vblank_queue);
4654 if (atomic_read(&rdev->irq.pflip[1]))
4655 radeon_crtc_handle_flip(rdev, 1);
4656 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4657 DRM_DEBUG("IH: D2 vblank\n");
4660 case 1: /* D2 vline */
4661 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4662 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4663 DRM_DEBUG("IH: D2 vline\n");
4667 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4671 case 3: /* D3 vblank/vline */
4673 case 0: /* D3 vblank */
4674 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4675 if (rdev->irq.crtc_vblank_int[2]) {
4676 drm_handle_vblank(rdev->ddev, 2);
4677 rdev->pm.vblank_sync = true;
4678 wake_up(&rdev->irq.vblank_queue);
4680 if (atomic_read(&rdev->irq.pflip[2]))
4681 radeon_crtc_handle_flip(rdev, 2);
4682 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4683 DRM_DEBUG("IH: D3 vblank\n");
4686 case 1: /* D3 vline */
4687 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4688 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4689 DRM_DEBUG("IH: D3 vline\n");
4693 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4697 case 4: /* D4 vblank/vline */
4699 case 0: /* D4 vblank */
4700 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4701 if (rdev->irq.crtc_vblank_int[3]) {
4702 drm_handle_vblank(rdev->ddev, 3);
4703 rdev->pm.vblank_sync = true;
4704 wake_up(&rdev->irq.vblank_queue);
4706 if (atomic_read(&rdev->irq.pflip[3]))
4707 radeon_crtc_handle_flip(rdev, 3);
4708 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4709 DRM_DEBUG("IH: D4 vblank\n");
4712 case 1: /* D4 vline */
4713 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4714 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4715 DRM_DEBUG("IH: D4 vline\n");
4719 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4723 case 5: /* D5 vblank/vline */
4725 case 0: /* D5 vblank */
4726 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4727 if (rdev->irq.crtc_vblank_int[4]) {
4728 drm_handle_vblank(rdev->ddev, 4);
4729 rdev->pm.vblank_sync = true;
4730 wake_up(&rdev->irq.vblank_queue);
4732 if (atomic_read(&rdev->irq.pflip[4]))
4733 radeon_crtc_handle_flip(rdev, 4);
4734 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4735 DRM_DEBUG("IH: D5 vblank\n");
4738 case 1: /* D5 vline */
4739 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4740 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4741 DRM_DEBUG("IH: D5 vline\n");
4745 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4749 case 6: /* D6 vblank/vline */
4751 case 0: /* D6 vblank */
4752 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4753 if (rdev->irq.crtc_vblank_int[5]) {
4754 drm_handle_vblank(rdev->ddev, 5);
4755 rdev->pm.vblank_sync = true;
4756 wake_up(&rdev->irq.vblank_queue);
4758 if (atomic_read(&rdev->irq.pflip[5]))
4759 radeon_crtc_handle_flip(rdev, 5);
4760 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4761 DRM_DEBUG("IH: D6 vblank\n");
4764 case 1: /* D6 vline */
4765 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4766 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4767 DRM_DEBUG("IH: D6 vline\n");
4771 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4775 case 42: /* HPD hotplug */
4778 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4779 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4780 queue_hotplug = true;
4781 DRM_DEBUG("IH: HPD1\n");
4785 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4786 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4787 queue_hotplug = true;
4788 DRM_DEBUG("IH: HPD2\n");
4792 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4793 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4794 queue_hotplug = true;
4795 DRM_DEBUG("IH: HPD3\n");
4799 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4800 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4801 queue_hotplug = true;
4802 DRM_DEBUG("IH: HPD4\n");
4806 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4807 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4808 queue_hotplug = true;
4809 DRM_DEBUG("IH: HPD5\n");
4813 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4814 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4815 queue_hotplug = true;
4816 DRM_DEBUG("IH: HPD6\n");
4820 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4827 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4828 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4830 DRM_DEBUG("IH: HDMI0\n");
4834 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4835 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4837 DRM_DEBUG("IH: HDMI1\n");
4841 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4842 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4844 DRM_DEBUG("IH: HDMI2\n");
4848 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4849 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4851 DRM_DEBUG("IH: HDMI3\n");
4855 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4856 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4858 DRM_DEBUG("IH: HDMI4\n");
4862 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4863 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4865 DRM_DEBUG("IH: HDMI5\n");
4869 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4873 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4874 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4878 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4879 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4880 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4881 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4883 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4885 cayman_vm_decode_fault(rdev, status, addr);
4886 /* reset addr and status */
4887 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4889 case 176: /* CP_INT in ring buffer */
4890 case 177: /* CP_INT in IB1 */
4891 case 178: /* CP_INT in IB2 */
4892 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4893 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4895 case 181: /* CP EOP event */
4896 DRM_DEBUG("IH: CP EOP\n");
4897 if (rdev->family >= CHIP_CAYMAN) {
4900 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4903 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4906 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4910 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4912 case 224: /* DMA trap event */
4913 DRM_DEBUG("IH: DMA trap\n");
4914 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4916 case 230: /* thermal low to high */
4917 DRM_DEBUG("IH: thermal low to high\n");
4918 rdev->pm.dpm.thermal.high_to_low = false;
4919 queue_thermal = true;
4921 case 231: /* thermal high to low */
4922 DRM_DEBUG("IH: thermal high to low\n");
4923 rdev->pm.dpm.thermal.high_to_low = true;
4924 queue_thermal = true;
4926 case 233: /* GUI IDLE */
4927 DRM_DEBUG("IH: GUI idle\n");
4929 case 244: /* DMA trap event */
4930 if (rdev->family >= CHIP_CAYMAN) {
4931 DRM_DEBUG("IH: DMA1 trap\n");
4932 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4936 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4940 /* wptr/rptr are in bytes! */
4942 rptr &= rdev->ih.ptr_mask;
4945 schedule_work(&rdev->hotplug_work);
4947 schedule_work(&rdev->audio_work);
4948 if (queue_thermal && rdev->pm.dpm_enabled)
4949 schedule_work(&rdev->pm.dpm.thermal.work);
4950 rdev->ih.rptr = rptr;
4951 WREG32(IH_RB_RPTR, rdev->ih.rptr);
4952 atomic_set(&rdev->ih.lock, 0);
4954 /* make sure wptr hasn't changed while processing */
4955 wptr = evergreen_get_ih_wptr(rdev);
4963 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
4965 * @rdev: radeon_device pointer
4966 * @fence: radeon fence object
4968 * Add a DMA fence packet to the ring to write
4969 * the fence seq number and DMA trap packet to generate
4970 * an interrupt if needed (evergreen-SI).
4972 void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
4973 struct radeon_fence *fence)
4975 struct radeon_ring *ring = &rdev->ring[fence->ring];
4976 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4977 /* write the fence */
4978 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
4979 radeon_ring_write(ring, addr & 0xfffffffc);
4980 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
4981 radeon_ring_write(ring, fence->seq);
4982 /* generate an interrupt */
4983 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
4985 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4986 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4987 radeon_ring_write(ring, 1);
4991 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
4993 * @rdev: radeon_device pointer
4994 * @ib: IB object to schedule
4996 * Schedule an IB in the DMA ring (evergreen).
4998 void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
4999 struct radeon_ib *ib)
5001 struct radeon_ring *ring = &rdev->ring[ib->ring];
5003 if (rdev->wb.enabled) {
5004 u32 next_rptr = ring->wptr + 4;
5005 while ((next_rptr & 7) != 5)
5008 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
5009 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5010 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
5011 radeon_ring_write(ring, next_rptr);
5014 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
5015 * Pad as necessary with NOPs.
5017 while ((ring->wptr & 7) != 5)
5018 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5019 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
5020 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
5021 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
5026 * evergreen_copy_dma - copy pages using the DMA engine
5028 * @rdev: radeon_device pointer
5029 * @src_offset: src GPU address
5030 * @dst_offset: dst GPU address
5031 * @num_gpu_pages: number of GPU pages to xfer
5032 * @fence: radeon fence object
5034 * Copy GPU paging using the DMA engine (evergreen-cayman).
5035 * Used by the radeon ttm implementation to move pages if
5036 * registered as the asic copy callback.
5038 int evergreen_copy_dma(struct radeon_device *rdev,
5039 uint64_t src_offset, uint64_t dst_offset,
5040 unsigned num_gpu_pages,
5041 struct radeon_fence **fence)
5043 struct radeon_semaphore *sem = NULL;
5044 int ring_index = rdev->asic->copy.dma_ring_index;
5045 struct radeon_ring *ring = &rdev->ring[ring_index];
5046 u32 size_in_dw, cur_size_in_dw;
5050 r = radeon_semaphore_create(rdev, &sem);
5052 DRM_ERROR("radeon: moving bo (%d).\n", r);
5056 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
5057 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
5058 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5060 DRM_ERROR("radeon: moving bo (%d).\n", r);
5061 radeon_semaphore_free(rdev, &sem, NULL);
5065 if (radeon_fence_need_sync(*fence, ring->idx)) {
5066 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5068 radeon_fence_note_sync(*fence, ring->idx);
5070 radeon_semaphore_free(rdev, &sem, NULL);
5073 for (i = 0; i < num_loops; i++) {
5074 cur_size_in_dw = size_in_dw;
5075 if (cur_size_in_dw > 0xFFFFF)
5076 cur_size_in_dw = 0xFFFFF;
5077 size_in_dw -= cur_size_in_dw;
5078 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
5079 radeon_ring_write(ring, dst_offset & 0xfffffffc);
5080 radeon_ring_write(ring, src_offset & 0xfffffffc);
5081 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5082 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5083 src_offset += cur_size_in_dw * 4;
5084 dst_offset += cur_size_in_dw * 4;
5087 r = radeon_fence_emit(rdev, fence, ring->idx);
5089 radeon_ring_unlock_undo(rdev, ring);
5093 radeon_ring_unlock_commit(rdev, ring);
5094 radeon_semaphore_free(rdev, &sem, *fence);
5099 static int evergreen_startup(struct radeon_device *rdev)
5101 struct radeon_ring *ring;
5104 /* enable pcie gen2 link */
5105 evergreen_pcie_gen2_enable(rdev);
5107 evergreen_program_aspm(rdev);
5109 evergreen_mc_program(rdev);
5111 if (ASIC_IS_DCE5(rdev)) {
5112 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5113 r = ni_init_microcode(rdev);
5115 DRM_ERROR("Failed to load firmware!\n");
5119 r = ni_mc_load_microcode(rdev);
5121 DRM_ERROR("Failed to load MC firmware!\n");
5125 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5126 r = r600_init_microcode(rdev);
5128 DRM_ERROR("Failed to load firmware!\n");
5134 r = r600_vram_scratch_init(rdev);
5138 if (rdev->flags & RADEON_IS_AGP) {
5139 evergreen_agp_enable(rdev);
5141 r = evergreen_pcie_gart_enable(rdev);
5145 evergreen_gpu_init(rdev);
5147 r = evergreen_blit_init(rdev);
5149 r600_blit_fini(rdev);
5150 rdev->asic->copy.copy = NULL;
5151 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
5154 /* allocate rlc buffers */
5155 if (rdev->flags & RADEON_IS_IGP) {
5156 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5157 rdev->rlc.reg_list_size = sumo_rlc_save_restore_register_list_size;
5158 rdev->rlc.cs_data = evergreen_cs_data;
5159 r = sumo_rlc_init(rdev);
5161 DRM_ERROR("Failed to init rlc BOs!\n");
5166 /* allocate wb buffer */
5167 r = radeon_wb_init(rdev);
5171 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5173 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5177 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5179 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5183 r = rv770_uvd_resume(rdev);
5185 r = radeon_fence_driver_start_ring(rdev,
5186 R600_RING_TYPE_UVD_INDEX);
5188 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5192 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5195 if (!rdev->irq.installed) {
5196 r = radeon_irq_kms_init(rdev);
5201 r = r600_irq_init(rdev);
5203 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5204 radeon_irq_kms_fini(rdev);
5207 evergreen_irq_set(rdev);
5209 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5210 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5211 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5212 0, 0xfffff, RADEON_CP_PACKET2);
5216 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5217 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5218 DMA_RB_RPTR, DMA_RB_WPTR,
5219 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5223 r = evergreen_cp_load_microcode(rdev);
5226 r = evergreen_cp_resume(rdev);
5229 r = r600_dma_resume(rdev);
5233 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5234 if (ring->ring_size) {
5235 r = radeon_ring_init(rdev, ring, ring->ring_size,
5236 R600_WB_UVD_RPTR_OFFSET,
5237 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5238 0, 0xfffff, RADEON_CP_PACKET2);
5240 r = r600_uvd_init(rdev);
5243 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5246 r = radeon_ib_pool_init(rdev);
5248 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5252 r = r600_audio_init(rdev);
5254 DRM_ERROR("radeon: audio init failed\n");
5261 int evergreen_resume(struct radeon_device *rdev)
5265 /* reset the asic, the gfx blocks are often in a bad state
5266 * after the driver is unloaded or after a resume
5268 if (radeon_asic_reset(rdev))
5269 dev_warn(rdev->dev, "GPU reset failed !\n");
5270 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5271 * posting will perform necessary task to bring back GPU into good
5275 atom_asic_init(rdev->mode_info.atom_context);
5277 /* init golden registers */
5278 evergreen_init_golden_registers(rdev);
5280 rdev->accel_working = true;
5281 r = evergreen_startup(rdev);
5283 DRM_ERROR("evergreen startup failed on resume\n");
5284 rdev->accel_working = false;
5292 int evergreen_suspend(struct radeon_device *rdev)
5294 r600_audio_fini(rdev);
5295 r600_uvd_stop(rdev);
5296 radeon_uvd_suspend(rdev);
5298 r600_dma_stop(rdev);
5299 evergreen_irq_suspend(rdev);
5300 radeon_wb_disable(rdev);
5301 evergreen_pcie_gart_disable(rdev);
5306 /* Plan is to move initialization in that function and use
5307 * helper function so that radeon_device_init pretty much
5308 * do nothing more than calling asic specific function. This
5309 * should also allow to remove a bunch of callback function
5312 int evergreen_init(struct radeon_device *rdev)
5317 if (!radeon_get_bios(rdev)) {
5318 if (ASIC_IS_AVIVO(rdev))
5321 /* Must be an ATOMBIOS */
5322 if (!rdev->is_atom_bios) {
5323 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5326 r = radeon_atombios_init(rdev);
5329 /* reset the asic, the gfx blocks are often in a bad state
5330 * after the driver is unloaded or after a resume
5332 if (radeon_asic_reset(rdev))
5333 dev_warn(rdev->dev, "GPU reset failed !\n");
5334 /* Post card if necessary */
5335 if (!radeon_card_posted(rdev)) {
5337 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5340 DRM_INFO("GPU not posted. posting now...\n");
5341 atom_asic_init(rdev->mode_info.atom_context);
5343 /* init golden registers */
5344 evergreen_init_golden_registers(rdev);
5345 /* Initialize scratch registers */
5346 r600_scratch_init(rdev);
5347 /* Initialize surface registers */
5348 radeon_surface_init(rdev);
5349 /* Initialize clocks */
5350 radeon_get_clock_info(rdev->ddev);
5352 r = radeon_fence_driver_init(rdev);
5355 /* initialize AGP */
5356 if (rdev->flags & RADEON_IS_AGP) {
5357 r = radeon_agp_init(rdev);
5359 radeon_agp_disable(rdev);
5361 /* initialize memory controller */
5362 r = evergreen_mc_init(rdev);
5365 /* Memory manager */
5366 r = radeon_bo_init(rdev);
5370 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5371 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5373 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5374 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5376 r = radeon_uvd_init(rdev);
5378 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5379 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5383 rdev->ih.ring_obj = NULL;
5384 r600_ih_ring_init(rdev, 64 * 1024);
5386 r = r600_pcie_gart_init(rdev);
5390 rdev->accel_working = true;
5391 r = evergreen_startup(rdev);
5393 dev_err(rdev->dev, "disabling GPU acceleration\n");
5395 r600_dma_fini(rdev);
5396 r600_irq_fini(rdev);
5397 if (rdev->flags & RADEON_IS_IGP)
5398 sumo_rlc_fini(rdev);
5399 radeon_wb_fini(rdev);
5400 radeon_ib_pool_fini(rdev);
5401 radeon_irq_kms_fini(rdev);
5402 evergreen_pcie_gart_fini(rdev);
5403 rdev->accel_working = false;
5406 /* Don't start up if the MC ucode is missing on BTC parts.
5407 * The default clocks and voltages before the MC ucode
5408 * is loaded are not suffient for advanced operations.
5410 if (ASIC_IS_DCE5(rdev)) {
5411 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5412 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5420 void evergreen_fini(struct radeon_device *rdev)
5422 r600_audio_fini(rdev);
5423 r600_blit_fini(rdev);
5425 r600_dma_fini(rdev);
5426 r600_irq_fini(rdev);
5427 if (rdev->flags & RADEON_IS_IGP)
5428 sumo_rlc_fini(rdev);
5429 radeon_wb_fini(rdev);
5430 radeon_ib_pool_fini(rdev);
5431 radeon_irq_kms_fini(rdev);
5432 evergreen_pcie_gart_fini(rdev);
5433 r600_uvd_stop(rdev);
5434 radeon_uvd_fini(rdev);
5435 r600_vram_scratch_fini(rdev);
5436 radeon_gem_fini(rdev);
5437 radeon_fence_driver_fini(rdev);
5438 radeon_agp_fini(rdev);
5439 radeon_bo_fini(rdev);
5440 radeon_atombios_fini(rdev);
5445 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5447 u32 link_width_cntl, speed_cntl;
5449 if (radeon_pcie_gen2 == 0)
5452 if (rdev->flags & RADEON_IS_IGP)
5455 if (!(rdev->flags & RADEON_IS_PCIE))
5458 /* x2 cards have a special sequence */
5459 if (ASIC_IS_X2(rdev))
5462 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5463 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5466 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5467 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5468 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5472 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5474 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5475 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5477 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5478 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5479 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5481 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5482 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5483 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5485 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5486 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5487 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5489 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5490 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5491 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5493 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5494 speed_cntl |= LC_GEN2_EN_STRAP;
5495 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5498 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5499 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5501 link_width_cntl |= LC_UPCONFIGURE_DIS;
5503 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5504 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5508 void evergreen_program_aspm(struct radeon_device *rdev)
5511 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5512 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5513 /* fusion_platform = true
5514 * if the system is a fusion system
5515 * (APU or DGPU in a fusion system).
5516 * todo: check if the system is a fusion platform.
5518 bool fusion_platform = false;
5520 if (radeon_aspm == 0)
5523 if (!(rdev->flags & RADEON_IS_PCIE))
5526 switch (rdev->family) {
5539 disable_l0s = false;
5543 if (rdev->flags & RADEON_IS_IGP)
5544 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5546 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5547 if (fusion_platform)
5552 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5554 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5555 if (fusion_platform)
5560 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5562 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5563 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5565 if (rdev->family >= CHIP_BARTS)
5566 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5568 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5572 if (rdev->family >= CHIP_BARTS)
5573 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5575 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5577 if (!disable_plloff_in_l1) {
5578 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5579 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5580 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5582 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5584 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5585 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5586 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5588 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5590 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5591 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5592 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5594 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5596 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5597 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5598 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5600 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5602 if (rdev->family >= CHIP_BARTS) {
5603 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5604 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5605 data |= PLL_RAMP_UP_TIME_0(4);
5607 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5609 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5610 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5611 data |= PLL_RAMP_UP_TIME_1(4);
5613 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5615 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5616 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5617 data |= PLL_RAMP_UP_TIME_0(4);
5619 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5621 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5622 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5623 data |= PLL_RAMP_UP_TIME_1(4);
5625 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5628 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5629 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5630 data |= LC_DYN_LANES_PWR_STATE(3);
5632 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5634 if (rdev->family >= CHIP_BARTS) {
5635 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5636 data &= ~LS2_EXIT_TIME_MASK;
5637 data |= LS2_EXIT_TIME(1);
5639 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5641 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5642 data &= ~LS2_EXIT_TIME_MASK;
5643 data |= LS2_EXIT_TIME(1);
5645 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5650 /* evergreen parts only */
5651 if (rdev->family < CHIP_BARTS)
5652 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5654 if (pcie_lc_cntl != pcie_lc_cntl_old)
5655 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);