projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/ci_dpm: Move 'ci_*()'s prototypes to shared header
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
c182615f 24
0f0de06c 25#include <linux/firmware.h>
0f0de06c 26#include <linux/module.h>
2ef79416
TZ		 27#include <linux/pci.h>
28#include <linux/slab.h>
c182615f 29
c182615f 30#include <drm/drm_vblank.h>
760285e7 31#include <drm/radeon_drm.h>
c182615f 32
43b3cd99 33#include "atom.h"
bd8cd539 34#include "clearstate_si.h"
4fe1999e 35#include "evergreen.h"
eb388a88 36#include "r600.h"
c182615f
SR		 37#include "radeon.h"
38#include "radeon_asic.h"
39#include "radeon_audio.h"
a0ceada6 40#include "radeon_ucode.h"
c182615f
SR		 41#include "si_blit_shaders.h"
42#include "sid.h"
43b3cd99 43
0f0de06c
AD		 44
45MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
46MODULE_FIRMWARE("radeon/TAHITI_me.bin");
47MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
48MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
1ebe9280 49MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
0f0de06c 50MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
a9e61410 51MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
629bd33c
AD		 52
53MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
54MODULE_FIRMWARE("radeon/tahiti_me.bin");
55MODULE_FIRMWARE("radeon/tahiti_ce.bin");
56MODULE_FIRMWARE("radeon/tahiti_mc.bin");
57MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
58MODULE_FIRMWARE("radeon/tahiti_smc.bin");
59
0f0de06c
AD		 60MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
61MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
62MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
63MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
1ebe9280 64MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
0f0de06c 65MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
a9e61410 66MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
629bd33c
AD		 67
68MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
69MODULE_FIRMWARE("radeon/pitcairn_me.bin");
70MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
71MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
72MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
73MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
861c7fde 74MODULE_FIRMWARE("radeon/pitcairn_k_smc.bin");
629bd33c 75
0f0de06c
AD		 76MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
77MODULE_FIRMWARE("radeon/VERDE_me.bin");
78MODULE_FIRMWARE("radeon/VERDE_ce.bin");
79MODULE_FIRMWARE("radeon/VERDE_mc.bin");
1ebe9280 80MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
0f0de06c 81MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
a9e61410 82MODULE_FIRMWARE("radeon/VERDE_smc.bin");
629bd33c
AD		 83
84MODULE_FIRMWARE("radeon/verde_pfp.bin");
85MODULE_FIRMWARE("radeon/verde_me.bin");
86MODULE_FIRMWARE("radeon/verde_ce.bin");
87MODULE_FIRMWARE("radeon/verde_mc.bin");
88MODULE_FIRMWARE("radeon/verde_rlc.bin");
89MODULE_FIRMWARE("radeon/verde_smc.bin");
861c7fde 90MODULE_FIRMWARE("radeon/verde_k_smc.bin");
629bd33c 91
bcc7f5d2
AD		 92MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
93MODULE_FIRMWARE("radeon/OLAND_me.bin");
94MODULE_FIRMWARE("radeon/OLAND_ce.bin");
95MODULE_FIRMWARE("radeon/OLAND_mc.bin");
1ebe9280 96MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
bcc7f5d2 97MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
a9e61410 98MODULE_FIRMWARE("radeon/OLAND_smc.bin");
629bd33c
AD		 99
100MODULE_FIRMWARE("radeon/oland_pfp.bin");
101MODULE_FIRMWARE("radeon/oland_me.bin");
102MODULE_FIRMWARE("radeon/oland_ce.bin");
103MODULE_FIRMWARE("radeon/oland_mc.bin");
104MODULE_FIRMWARE("radeon/oland_rlc.bin");
105MODULE_FIRMWARE("radeon/oland_smc.bin");
861c7fde 106MODULE_FIRMWARE("radeon/oland_k_smc.bin");
629bd33c 107
c04c00b4
AD		 108MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
109MODULE_FIRMWARE("radeon/HAINAN_me.bin");
110MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
111MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
1ebe9280 112MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
c04c00b4 113MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
a9e61410 114MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
0f0de06c 115
629bd33c
AD		 116MODULE_FIRMWARE("radeon/hainan_pfp.bin");
117MODULE_FIRMWARE("radeon/hainan_me.bin");
118MODULE_FIRMWARE("radeon/hainan_ce.bin");
119MODULE_FIRMWARE("radeon/hainan_mc.bin");
120MODULE_FIRMWARE("radeon/hainan_rlc.bin");
121MODULE_FIRMWARE("radeon/hainan_smc.bin");
861c7fde 122MODULE_FIRMWARE("radeon/hainan_k_smc.bin");
4e6e98b1 123MODULE_FIRMWARE("radeon/banks_k_2_smc.bin");
629bd33c 124
ef736d39
AD		 125MODULE_FIRMWARE("radeon/si58_mc.bin");
126
65fcf668 127static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
b9d305df 128static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 129static void si_program_aspm(struct radeon_device *rdev);
1fd11777
AD		 130extern void sumo_rlc_fini(struct radeon_device *rdev);
131extern int sumo_rlc_init(struct radeon_device *rdev);
811e4d58
AD		 132static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
133 bool enable);
4a5c8ea5
AD		 134static void si_init_pg(struct radeon_device *rdev);
135static void si_init_cg(struct radeon_device *rdev);
a6f4ae8d
AD		 136static void si_fini_pg(struct radeon_device *rdev);
137static void si_fini_cg(struct radeon_device *rdev);
138static void si_rlc_stop(struct radeon_device *rdev);
0a96d72b 139
4cd096dd
L		 140static const u32 crtc_offsets[] =
141{
142 EVERGREEN_CRTC0_REGISTER_OFFSET,
143 EVERGREEN_CRTC1_REGISTER_OFFSET,
144 EVERGREEN_CRTC2_REGISTER_OFFSET,
145 EVERGREEN_CRTC3_REGISTER_OFFSET,
146 EVERGREEN_CRTC4_REGISTER_OFFSET,
147 EVERGREEN_CRTC5_REGISTER_OFFSET
148};
149
150static const u32 si_disp_int_status[] =
151{
152 DISP_INTERRUPT_STATUS,
153 DISP_INTERRUPT_STATUS_CONTINUE,
154 DISP_INTERRUPT_STATUS_CONTINUE2,
155 DISP_INTERRUPT_STATUS_CONTINUE3,
156 DISP_INTERRUPT_STATUS_CONTINUE4,
157 DISP_INTERRUPT_STATUS_CONTINUE5
158};
159
160#define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
161#define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
162#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
163
6d8cf000
AD		 164static const u32 verde_rlc_save_restore_register_list[] =
165{
166 (0x8000 << 16) | (0x98f4 >> 2),
167 0x00000000,
168 (0x8040 << 16) | (0x98f4 >> 2),
169 0x00000000,
170 (0x8000 << 16) | (0xe80 >> 2),
171 0x00000000,
172 (0x8040 << 16) | (0xe80 >> 2),
173 0x00000000,
174 (0x8000 << 16) | (0x89bc >> 2),
175 0x00000000,
176 (0x8040 << 16) | (0x89bc >> 2),
177 0x00000000,
178 (0x8000 << 16) | (0x8c1c >> 2),
179 0x00000000,
180 (0x8040 << 16) | (0x8c1c >> 2),
181 0x00000000,
182 (0x9c00 << 16) | (0x98f0 >> 2),
183 0x00000000,
184 (0x9c00 << 16) | (0xe7c >> 2),
185 0x00000000,
186 (0x8000 << 16) | (0x9148 >> 2),
187 0x00000000,
188 (0x8040 << 16) | (0x9148 >> 2),
189 0x00000000,
190 (0x9c00 << 16) | (0x9150 >> 2),
191 0x00000000,
192 (0x9c00 << 16) | (0x897c >> 2),
193 0x00000000,
194 (0x9c00 << 16) | (0x8d8c >> 2),
195 0x00000000,
196 (0x9c00 << 16) | (0xac54 >> 2),
197 0X00000000,
198 0x3,
199 (0x9c00 << 16) | (0x98f8 >> 2),
200 0x00000000,
201 (0x9c00 << 16) | (0x9910 >> 2),
202 0x00000000,
203 (0x9c00 << 16) | (0x9914 >> 2),
204 0x00000000,
205 (0x9c00 << 16) | (0x9918 >> 2),
206 0x00000000,
207 (0x9c00 << 16) | (0x991c >> 2),
208 0x00000000,
209 (0x9c00 << 16) | (0x9920 >> 2),
210 0x00000000,
211 (0x9c00 << 16) | (0x9924 >> 2),
212 0x00000000,
213 (0x9c00 << 16) | (0x9928 >> 2),
214 0x00000000,
215 (0x9c00 << 16) | (0x992c >> 2),
216 0x00000000,
217 (0x9c00 << 16) | (0x9930 >> 2),
218 0x00000000,
219 (0x9c00 << 16) | (0x9934 >> 2),
220 0x00000000,
221 (0x9c00 << 16) | (0x9938 >> 2),
222 0x00000000,
223 (0x9c00 << 16) | (0x993c >> 2),
224 0x00000000,
225 (0x9c00 << 16) | (0x9940 >> 2),
226 0x00000000,
227 (0x9c00 << 16) | (0x9944 >> 2),
228 0x00000000,
229 (0x9c00 << 16) | (0x9948 >> 2),
230 0x00000000,
231 (0x9c00 << 16) | (0x994c >> 2),
232 0x00000000,
233 (0x9c00 << 16) | (0x9950 >> 2),
234 0x00000000,
235 (0x9c00 << 16) | (0x9954 >> 2),
236 0x00000000,
237 (0x9c00 << 16) | (0x9958 >> 2),
238 0x00000000,
239 (0x9c00 << 16) | (0x995c >> 2),
240 0x00000000,
241 (0x9c00 << 16) | (0x9960 >> 2),
242 0x00000000,
243 (0x9c00 << 16) | (0x9964 >> 2),
244 0x00000000,
245 (0x9c00 << 16) | (0x9968 >> 2),
246 0x00000000,
247 (0x9c00 << 16) | (0x996c >> 2),
248 0x00000000,
249 (0x9c00 << 16) | (0x9970 >> 2),
250 0x00000000,
251 (0x9c00 << 16) | (0x9974 >> 2),
252 0x00000000,
253 (0x9c00 << 16) | (0x9978 >> 2),
254 0x00000000,
255 (0x9c00 << 16) | (0x997c >> 2),
256 0x00000000,
257 (0x9c00 << 16) | (0x9980 >> 2),
258 0x00000000,
259 (0x9c00 << 16) | (0x9984 >> 2),
260 0x00000000,
261 (0x9c00 << 16) | (0x9988 >> 2),
262 0x00000000,
263 (0x9c00 << 16) | (0x998c >> 2),
264 0x00000000,
265 (0x9c00 << 16) | (0x8c00 >> 2),
266 0x00000000,
267 (0x9c00 << 16) | (0x8c14 >> 2),
268 0x00000000,
269 (0x9c00 << 16) | (0x8c04 >> 2),
270 0x00000000,
271 (0x9c00 << 16) | (0x8c08 >> 2),
272 0x00000000,
273 (0x8000 << 16) | (0x9b7c >> 2),
274 0x00000000,
275 (0x8040 << 16) | (0x9b7c >> 2),
276 0x00000000,
277 (0x8000 << 16) | (0xe84 >> 2),
278 0x00000000,
279 (0x8040 << 16) | (0xe84 >> 2),
280 0x00000000,
281 (0x8000 << 16) | (0x89c0 >> 2),
282 0x00000000,
283 (0x8040 << 16) | (0x89c0 >> 2),
284 0x00000000,
285 (0x8000 << 16) | (0x914c >> 2),
286 0x00000000,
287 (0x8040 << 16) | (0x914c >> 2),
288 0x00000000,
289 (0x8000 << 16) | (0x8c20 >> 2),
290 0x00000000,
291 (0x8040 << 16) | (0x8c20 >> 2),
292 0x00000000,
293 (0x8000 << 16) | (0x9354 >> 2),
294 0x00000000,
295 (0x8040 << 16) | (0x9354 >> 2),
296 0x00000000,
297 (0x9c00 << 16) | (0x9060 >> 2),
298 0x00000000,
299 (0x9c00 << 16) | (0x9364 >> 2),
300 0x00000000,
301 (0x9c00 << 16) | (0x9100 >> 2),
302 0x00000000,
303 (0x9c00 << 16) | (0x913c >> 2),
304 0x00000000,
305 (0x8000 << 16) | (0x90e0 >> 2),
306 0x00000000,
307 (0x8000 << 16) | (0x90e4 >> 2),
308 0x00000000,
309 (0x8000 << 16) | (0x90e8 >> 2),
310 0x00000000,
311 (0x8040 << 16) | (0x90e0 >> 2),
312 0x00000000,
313 (0x8040 << 16) | (0x90e4 >> 2),
314 0x00000000,
315 (0x8040 << 16) | (0x90e8 >> 2),
316 0x00000000,
317 (0x9c00 << 16) | (0x8bcc >> 2),
318 0x00000000,
319 (0x9c00 << 16) | (0x8b24 >> 2),
320 0x00000000,
321 (0x9c00 << 16) | (0x88c4 >> 2),
322 0x00000000,
323 (0x9c00 << 16) | (0x8e50 >> 2),
324 0x00000000,
325 (0x9c00 << 16) | (0x8c0c >> 2),
326 0x00000000,
327 (0x9c00 << 16) | (0x8e58 >> 2),
328 0x00000000,
329 (0x9c00 << 16) | (0x8e5c >> 2),
330 0x00000000,
331 (0x9c00 << 16) | (0x9508 >> 2),
332 0x00000000,
333 (0x9c00 << 16) | (0x950c >> 2),
334 0x00000000,
335 (0x9c00 << 16) | (0x9494 >> 2),
336 0x00000000,
337 (0x9c00 << 16) | (0xac0c >> 2),
338 0x00000000,
339 (0x9c00 << 16) | (0xac10 >> 2),
340 0x00000000,
341 (0x9c00 << 16) | (0xac14 >> 2),
342 0x00000000,
343 (0x9c00 << 16) | (0xae00 >> 2),
344 0x00000000,
345 (0x9c00 << 16) | (0xac08 >> 2),
346 0x00000000,
347 (0x9c00 << 16) | (0x88d4 >> 2),
348 0x00000000,
349 (0x9c00 << 16) | (0x88c8 >> 2),
350 0x00000000,
351 (0x9c00 << 16) | (0x88cc >> 2),
352 0x00000000,
353 (0x9c00 << 16) | (0x89b0 >> 2),
354 0x00000000,
355 (0x9c00 << 16) | (0x8b10 >> 2),
356 0x00000000,
357 (0x9c00 << 16) | (0x8a14 >> 2),
358 0x00000000,
359 (0x9c00 << 16) | (0x9830 >> 2),
360 0x00000000,
361 (0x9c00 << 16) | (0x9834 >> 2),
362 0x00000000,
363 (0x9c00 << 16) | (0x9838 >> 2),
364 0x00000000,
365 (0x9c00 << 16) | (0x9a10 >> 2),
366 0x00000000,
367 (0x8000 << 16) | (0x9870 >> 2),
368 0x00000000,
369 (0x8000 << 16) | (0x9874 >> 2),
370 0x00000000,
371 (0x8001 << 16) | (0x9870 >> 2),
372 0x00000000,
373 (0x8001 << 16) | (0x9874 >> 2),
374 0x00000000,
375 (0x8040 << 16) | (0x9870 >> 2),
376 0x00000000,
377 (0x8040 << 16) | (0x9874 >> 2),
378 0x00000000,
379 (0x8041 << 16) | (0x9870 >> 2),
380 0x00000000,
381 (0x8041 << 16) | (0x9874 >> 2),
382 0x00000000,
383 0x00000000
384};
385
205996c0
AD		 386static const u32 tahiti_golden_rlc_registers[] =
387{
388 0xc424, 0xffffffff, 0x00601005,
389 0xc47c, 0xffffffff, 0x10104040,
390 0xc488, 0xffffffff, 0x0100000a,
391 0xc314, 0xffffffff, 0x00000800,
392 0xc30c, 0xffffffff, 0x800000f4,
393 0xf4a8, 0xffffffff, 0x00000000
394};
395
396static const u32 tahiti_golden_registers[] =
397{
398 0x9a10, 0x00010000, 0x00018208,
399 0x9830, 0xffffffff, 0x00000000,
400 0x9834, 0xf00fffff, 0x00000400,
401 0x9838, 0x0002021c, 0x00020200,
402 0xc78, 0x00000080, 0x00000000,
403 0xd030, 0x000300c0, 0x00800040,
404 0xd830, 0x000300c0, 0x00800040,
405 0x5bb0, 0x000000f0, 0x00000070,
406 0x5bc0, 0x00200000, 0x50100000,
407 0x7030, 0x31000311, 0x00000011,
408 0x277c, 0x00000003, 0x000007ff,
409 0x240c, 0x000007ff, 0x00000000,
410 0x8a14, 0xf000001f, 0x00000007,
411 0x8b24, 0xffffffff, 0x00ffffff,
412 0x8b10, 0x0000ff0f, 0x00000000,
413 0x28a4c, 0x07ffffff, 0x4e000000,
414 0x28350, 0x3f3f3fff, 0x2a00126a,
415 0x30, 0x000000ff, 0x0040,
416 0x34, 0x00000040, 0x00004040,
417 0x9100, 0x07ffffff, 0x03000000,
418 0x8e88, 0x01ff1f3f, 0x00000000,
419 0x8e84, 0x01ff1f3f, 0x00000000,
420 0x9060, 0x0000007f, 0x00000020,
421 0x9508, 0x00010000, 0x00010000,
422 0xac14, 0x00000200, 0x000002fb,
423 0xac10, 0xffffffff, 0x0000543b,
424 0xac0c, 0xffffffff, 0xa9210876,
425 0x88d0, 0xffffffff, 0x000fff40,
426 0x88d4, 0x0000001f, 0x00000010,
427 0x1410, 0x20000000, 0x20fffed8,
428 0x15c0, 0x000c0fc0, 0x000c0400
429};
430
431static const u32 tahiti_golden_registers2[] =
432{
433 0xc64, 0x00000001, 0x00000001
434};
435
436static const u32 pitcairn_golden_rlc_registers[] =
437{
438 0xc424, 0xffffffff, 0x00601004,
439 0xc47c, 0xffffffff, 0x10102020,
440 0xc488, 0xffffffff, 0x01000020,
441 0xc314, 0xffffffff, 0x00000800,
442 0xc30c, 0xffffffff, 0x800000a4
443};
444
445static const u32 pitcairn_golden_registers[] =
446{
447 0x9a10, 0x00010000, 0x00018208,
448 0x9830, 0xffffffff, 0x00000000,
449 0x9834, 0xf00fffff, 0x00000400,
450 0x9838, 0x0002021c, 0x00020200,
451 0xc78, 0x00000080, 0x00000000,
452 0xd030, 0x000300c0, 0x00800040,
453 0xd830, 0x000300c0, 0x00800040,
454 0x5bb0, 0x000000f0, 0x00000070,
455 0x5bc0, 0x00200000, 0x50100000,
456 0x7030, 0x31000311, 0x00000011,
457 0x2ae4, 0x00073ffe, 0x000022a2,
458 0x240c, 0x000007ff, 0x00000000,
459 0x8a14, 0xf000001f, 0x00000007,
460 0x8b24, 0xffffffff, 0x00ffffff,
461 0x8b10, 0x0000ff0f, 0x00000000,
462 0x28a4c, 0x07ffffff, 0x4e000000,
463 0x28350, 0x3f3f3fff, 0x2a00126a,
464 0x30, 0x000000ff, 0x0040,
465 0x34, 0x00000040, 0x00004040,
466 0x9100, 0x07ffffff, 0x03000000,
467 0x9060, 0x0000007f, 0x00000020,
468 0x9508, 0x00010000, 0x00010000,
469 0xac14, 0x000003ff, 0x000000f7,
470 0xac10, 0xffffffff, 0x00000000,
471 0xac0c, 0xffffffff, 0x32761054,
472 0x88d4, 0x0000001f, 0x00000010,
473 0x15c0, 0x000c0fc0, 0x000c0400
474};
475
476static const u32 verde_golden_rlc_registers[] =
477{
478 0xc424, 0xffffffff, 0x033f1005,
479 0xc47c, 0xffffffff, 0x10808020,
480 0xc488, 0xffffffff, 0x00800008,
481 0xc314, 0xffffffff, 0x00001000,
482 0xc30c, 0xffffffff, 0x80010014
483};
484
485static const u32 verde_golden_registers[] =
486{
487 0x9a10, 0x00010000, 0x00018208,
488 0x9830, 0xffffffff, 0x00000000,
489 0x9834, 0xf00fffff, 0x00000400,
490 0x9838, 0x0002021c, 0x00020200,
491 0xc78, 0x00000080, 0x00000000,
492 0xd030, 0x000300c0, 0x00800040,
493 0xd030, 0x000300c0, 0x00800040,
494 0xd830, 0x000300c0, 0x00800040,
495 0xd830, 0x000300c0, 0x00800040,
496 0x5bb0, 0x000000f0, 0x00000070,
497 0x5bc0, 0x00200000, 0x50100000,
498 0x7030, 0x31000311, 0x00000011,
499 0x2ae4, 0x00073ffe, 0x000022a2,
500 0x2ae4, 0x00073ffe, 0x000022a2,
501 0x2ae4, 0x00073ffe, 0x000022a2,
502 0x240c, 0x000007ff, 0x00000000,
503 0x240c, 0x000007ff, 0x00000000,
504 0x240c, 0x000007ff, 0x00000000,
505 0x8a14, 0xf000001f, 0x00000007,
506 0x8a14, 0xf000001f, 0x00000007,
507 0x8a14, 0xf000001f, 0x00000007,
508 0x8b24, 0xffffffff, 0x00ffffff,
509 0x8b10, 0x0000ff0f, 0x00000000,
510 0x28a4c, 0x07ffffff, 0x4e000000,
511 0x28350, 0x3f3f3fff, 0x0000124a,
512 0x28350, 0x3f3f3fff, 0x0000124a,
513 0x28350, 0x3f3f3fff, 0x0000124a,
514 0x30, 0x000000ff, 0x0040,
515 0x34, 0x00000040, 0x00004040,
516 0x9100, 0x07ffffff, 0x03000000,
517 0x9100, 0x07ffffff, 0x03000000,
518 0x8e88, 0x01ff1f3f, 0x00000000,
519 0x8e88, 0x01ff1f3f, 0x00000000,
520 0x8e88, 0x01ff1f3f, 0x00000000,
521 0x8e84, 0x01ff1f3f, 0x00000000,
522 0x8e84, 0x01ff1f3f, 0x00000000,
523 0x8e84, 0x01ff1f3f, 0x00000000,
524 0x9060, 0x0000007f, 0x00000020,
525 0x9508, 0x00010000, 0x00010000,
526 0xac14, 0x000003ff, 0x00000003,
527 0xac14, 0x000003ff, 0x00000003,
528 0xac14, 0x000003ff, 0x00000003,
529 0xac10, 0xffffffff, 0x00000000,
530 0xac10, 0xffffffff, 0x00000000,
531 0xac10, 0xffffffff, 0x00000000,
532 0xac0c, 0xffffffff, 0x00001032,
533 0xac0c, 0xffffffff, 0x00001032,
534 0xac0c, 0xffffffff, 0x00001032,
535 0x88d4, 0x0000001f, 0x00000010,
536 0x88d4, 0x0000001f, 0x00000010,
537 0x88d4, 0x0000001f, 0x00000010,
538 0x15c0, 0x000c0fc0, 0x000c0400
539};
540
541static const u32 oland_golden_rlc_registers[] =
542{
543 0xc424, 0xffffffff, 0x00601005,
544 0xc47c, 0xffffffff, 0x10104040,
545 0xc488, 0xffffffff, 0x0100000a,
546 0xc314, 0xffffffff, 0x00000800,
547 0xc30c, 0xffffffff, 0x800000f4
548};
549
550static const u32 oland_golden_registers[] =
551{
552 0x9a10, 0x00010000, 0x00018208,
553 0x9830, 0xffffffff, 0x00000000,
554 0x9834, 0xf00fffff, 0x00000400,
555 0x9838, 0x0002021c, 0x00020200,
556 0xc78, 0x00000080, 0x00000000,
557 0xd030, 0x000300c0, 0x00800040,
558 0xd830, 0x000300c0, 0x00800040,
559 0x5bb0, 0x000000f0, 0x00000070,
560 0x5bc0, 0x00200000, 0x50100000,
561 0x7030, 0x31000311, 0x00000011,
562 0x2ae4, 0x00073ffe, 0x000022a2,
563 0x240c, 0x000007ff, 0x00000000,
564 0x8a14, 0xf000001f, 0x00000007,
565 0x8b24, 0xffffffff, 0x00ffffff,
566 0x8b10, 0x0000ff0f, 0x00000000,
567 0x28a4c, 0x07ffffff, 0x4e000000,
568 0x28350, 0x3f3f3fff, 0x00000082,
569 0x30, 0x000000ff, 0x0040,
570 0x34, 0x00000040, 0x00004040,
571 0x9100, 0x07ffffff, 0x03000000,
572 0x9060, 0x0000007f, 0x00000020,
573 0x9508, 0x00010000, 0x00010000,
574 0xac14, 0x000003ff, 0x000000f3,
575 0xac10, 0xffffffff, 0x00000000,
576 0xac0c, 0xffffffff, 0x00003210,
577 0x88d4, 0x0000001f, 0x00000010,
578 0x15c0, 0x000c0fc0, 0x000c0400
579};
580
fffbdda4
AD		 581static const u32 hainan_golden_registers[] =
582{
583 0x9a10, 0x00010000, 0x00018208,
584 0x9830, 0xffffffff, 0x00000000,
585 0x9834, 0xf00fffff, 0x00000400,
586 0x9838, 0x0002021c, 0x00020200,
587 0xd0c0, 0xff000fff, 0x00000100,
588 0xd030, 0x000300c0, 0x00800040,
589 0xd8c0, 0xff000fff, 0x00000100,
590 0xd830, 0x000300c0, 0x00800040,
591 0x2ae4, 0x00073ffe, 0x000022a2,
592 0x240c, 0x000007ff, 0x00000000,
593 0x8a14, 0xf000001f, 0x00000007,
594 0x8b24, 0xffffffff, 0x00ffffff,
595 0x8b10, 0x0000ff0f, 0x00000000,
596 0x28a4c, 0x07ffffff, 0x4e000000,
597 0x28350, 0x3f3f3fff, 0x00000000,
598 0x30, 0x000000ff, 0x0040,
599 0x34, 0x00000040, 0x00004040,
600 0x9100, 0x03e00000, 0x03600000,
601 0x9060, 0x0000007f, 0x00000020,
602 0x9508, 0x00010000, 0x00010000,
603 0xac14, 0x000003ff, 0x000000f1,
604 0xac10, 0xffffffff, 0x00000000,
605 0xac0c, 0xffffffff, 0x00003210,
606 0x88d4, 0x0000001f, 0x00000010,
607 0x15c0, 0x000c0fc0, 0x000c0400
608};
609
610static const u32 hainan_golden_registers2[] =
611{
612 0x98f8, 0xffffffff, 0x02010001
613};
614
205996c0
AD		 615static const u32 tahiti_mgcg_cgcg_init[] =
616{
617 0xc400, 0xffffffff, 0xfffffffc,
618 0x802c, 0xffffffff, 0xe0000000,
619 0x9a60, 0xffffffff, 0x00000100,
620 0x92a4, 0xffffffff, 0x00000100,
621 0xc164, 0xffffffff, 0x00000100,
622 0x9774, 0xffffffff, 0x00000100,
623 0x8984, 0xffffffff, 0x06000100,
624 0x8a18, 0xffffffff, 0x00000100,
625 0x92a0, 0xffffffff, 0x00000100,
626 0xc380, 0xffffffff, 0x00000100,
627 0x8b28, 0xffffffff, 0x00000100,
628 0x9144, 0xffffffff, 0x00000100,
629 0x8d88, 0xffffffff, 0x00000100,
630 0x8d8c, 0xffffffff, 0x00000100,
631 0x9030, 0xffffffff, 0x00000100,
632 0x9034, 0xffffffff, 0x00000100,
633 0x9038, 0xffffffff, 0x00000100,
634 0x903c, 0xffffffff, 0x00000100,
635 0xad80, 0xffffffff, 0x00000100,
636 0xac54, 0xffffffff, 0x00000100,
637 0x897c, 0xffffffff, 0x06000100,
638 0x9868, 0xffffffff, 0x00000100,
639 0x9510, 0xffffffff, 0x00000100,
640 0xaf04, 0xffffffff, 0x00000100,
641 0xae04, 0xffffffff, 0x00000100,
642 0x949c, 0xffffffff, 0x00000100,
643 0x802c, 0xffffffff, 0xe0000000,
644 0x9160, 0xffffffff, 0x00010000,
645 0x9164, 0xffffffff, 0x00030002,
646 0x9168, 0xffffffff, 0x00040007,
647 0x916c, 0xffffffff, 0x00060005,
648 0x9170, 0xffffffff, 0x00090008,
649 0x9174, 0xffffffff, 0x00020001,
650 0x9178, 0xffffffff, 0x00040003,
651 0x917c, 0xffffffff, 0x00000007,
652 0x9180, 0xffffffff, 0x00060005,
653 0x9184, 0xffffffff, 0x00090008,
654 0x9188, 0xffffffff, 0x00030002,
655 0x918c, 0xffffffff, 0x00050004,
656 0x9190, 0xffffffff, 0x00000008,
657 0x9194, 0xffffffff, 0x00070006,
658 0x9198, 0xffffffff, 0x000a0009,
659 0x919c, 0xffffffff, 0x00040003,
660 0x91a0, 0xffffffff, 0x00060005,
661 0x91a4, 0xffffffff, 0x00000009,
662 0x91a8, 0xffffffff, 0x00080007,
663 0x91ac, 0xffffffff, 0x000b000a,
664 0x91b0, 0xffffffff, 0x00050004,
665 0x91b4, 0xffffffff, 0x00070006,
666 0x91b8, 0xffffffff, 0x0008000b,
667 0x91bc, 0xffffffff, 0x000a0009,
668 0x91c0, 0xffffffff, 0x000d000c,
669 0x91c4, 0xffffffff, 0x00060005,
670 0x91c8, 0xffffffff, 0x00080007,
671 0x91cc, 0xffffffff, 0x0000000b,
672 0x91d0, 0xffffffff, 0x000a0009,
673 0x91d4, 0xffffffff, 0x000d000c,
674 0x91d8, 0xffffffff, 0x00070006,
675 0x91dc, 0xffffffff, 0x00090008,
676 0x91e0, 0xffffffff, 0x0000000c,
677 0x91e4, 0xffffffff, 0x000b000a,
678 0x91e8, 0xffffffff, 0x000e000d,
679 0x91ec, 0xffffffff, 0x00080007,
680 0x91f0, 0xffffffff, 0x000a0009,
681 0x91f4, 0xffffffff, 0x0000000d,
682 0x91f8, 0xffffffff, 0x000c000b,
683 0x91fc, 0xffffffff, 0x000f000e,
684 0x9200, 0xffffffff, 0x00090008,
685 0x9204, 0xffffffff, 0x000b000a,
686 0x9208, 0xffffffff, 0x000c000f,
687 0x920c, 0xffffffff, 0x000e000d,
688 0x9210, 0xffffffff, 0x00110010,
689 0x9214, 0xffffffff, 0x000a0009,
690 0x9218, 0xffffffff, 0x000c000b,
691 0x921c, 0xffffffff, 0x0000000f,
692 0x9220, 0xffffffff, 0x000e000d,
693 0x9224, 0xffffffff, 0x00110010,
694 0x9228, 0xffffffff, 0x000b000a,
695 0x922c, 0xffffffff, 0x000d000c,
696 0x9230, 0xffffffff, 0x00000010,
697 0x9234, 0xffffffff, 0x000f000e,
698 0x9238, 0xffffffff, 0x00120011,
699 0x923c, 0xffffffff, 0x000c000b,
700 0x9240, 0xffffffff, 0x000e000d,
701 0x9244, 0xffffffff, 0x00000011,
702 0x9248, 0xffffffff, 0x0010000f,
703 0x924c, 0xffffffff, 0x00130012,
704 0x9250, 0xffffffff, 0x000d000c,
705 0x9254, 0xffffffff, 0x000f000e,
706 0x9258, 0xffffffff, 0x00100013,
707 0x925c, 0xffffffff, 0x00120011,
708 0x9260, 0xffffffff, 0x00150014,
709 0x9264, 0xffffffff, 0x000e000d,
710 0x9268, 0xffffffff, 0x0010000f,
711 0x926c, 0xffffffff, 0x00000013,
712 0x9270, 0xffffffff, 0x00120011,
713 0x9274, 0xffffffff, 0x00150014,
714 0x9278, 0xffffffff, 0x000f000e,
715 0x927c, 0xffffffff, 0x00110010,
716 0x9280, 0xffffffff, 0x00000014,
717 0x9284, 0xffffffff, 0x00130012,
718 0x9288, 0xffffffff, 0x00160015,
719 0x928c, 0xffffffff, 0x0010000f,
720 0x9290, 0xffffffff, 0x00120011,
721 0x9294, 0xffffffff, 0x00000015,
722 0x9298, 0xffffffff, 0x00140013,
723 0x929c, 0xffffffff, 0x00170016,
724 0x9150, 0xffffffff, 0x96940200,
725 0x8708, 0xffffffff, 0x00900100,
726 0xc478, 0xffffffff, 0x00000080,
727 0xc404, 0xffffffff, 0x0020003f,
728 0x30, 0xffffffff, 0x0000001c,
729 0x34, 0x000f0000, 0x000f0000,
730 0x160c, 0xffffffff, 0x00000100,
731 0x1024, 0xffffffff, 0x00000100,
732 0x102c, 0x00000101, 0x00000000,
733 0x20a8, 0xffffffff, 0x00000104,
734 0x264c, 0x000c0000, 0x000c0000,
735 0x2648, 0x000c0000, 0x000c0000,
736 0x55e4, 0xff000fff, 0x00000100,
737 0x55e8, 0x00000001, 0x00000001,
738 0x2f50, 0x00000001, 0x00000001,
739 0x30cc, 0xc0000fff, 0x00000104,
740 0xc1e4, 0x00000001, 0x00000001,
741 0xd0c0, 0xfffffff0, 0x00000100,
742 0xd8c0, 0xfffffff0, 0x00000100
743};
744
745static const u32 pitcairn_mgcg_cgcg_init[] =
746{
747 0xc400, 0xffffffff, 0xfffffffc,
748 0x802c, 0xffffffff, 0xe0000000,
749 0x9a60, 0xffffffff, 0x00000100,
750 0x92a4, 0xffffffff, 0x00000100,
751 0xc164, 0xffffffff, 0x00000100,
752 0x9774, 0xffffffff, 0x00000100,
753 0x8984, 0xffffffff, 0x06000100,
754 0x8a18, 0xffffffff, 0x00000100,
755 0x92a0, 0xffffffff, 0x00000100,
756 0xc380, 0xffffffff, 0x00000100,
757 0x8b28, 0xffffffff, 0x00000100,
758 0x9144, 0xffffffff, 0x00000100,
759 0x8d88, 0xffffffff, 0x00000100,
760 0x8d8c, 0xffffffff, 0x00000100,
761 0x9030, 0xffffffff, 0x00000100,
762 0x9034, 0xffffffff, 0x00000100,
763 0x9038, 0xffffffff, 0x00000100,
764 0x903c, 0xffffffff, 0x00000100,
765 0xad80, 0xffffffff, 0x00000100,
766 0xac54, 0xffffffff, 0x00000100,
767 0x897c, 0xffffffff, 0x06000100,
768 0x9868, 0xffffffff, 0x00000100,
769 0x9510, 0xffffffff, 0x00000100,
770 0xaf04, 0xffffffff, 0x00000100,
771 0xae04, 0xffffffff, 0x00000100,
772 0x949c, 0xffffffff, 0x00000100,
773 0x802c, 0xffffffff, 0xe0000000,
774 0x9160, 0xffffffff, 0x00010000,
775 0x9164, 0xffffffff, 0x00030002,
776 0x9168, 0xffffffff, 0x00040007,
777 0x916c, 0xffffffff, 0x00060005,
778 0x9170, 0xffffffff, 0x00090008,
779 0x9174, 0xffffffff, 0x00020001,
780 0x9178, 0xffffffff, 0x00040003,
781 0x917c, 0xffffffff, 0x00000007,
782 0x9180, 0xffffffff, 0x00060005,
783 0x9184, 0xffffffff, 0x00090008,
784 0x9188, 0xffffffff, 0x00030002,
785 0x918c, 0xffffffff, 0x00050004,
786 0x9190, 0xffffffff, 0x00000008,
787 0x9194, 0xffffffff, 0x00070006,
788 0x9198, 0xffffffff, 0x000a0009,
789 0x919c, 0xffffffff, 0x00040003,
790 0x91a0, 0xffffffff, 0x00060005,
791 0x91a4, 0xffffffff, 0x00000009,
792 0x91a8, 0xffffffff, 0x00080007,
793 0x91ac, 0xffffffff, 0x000b000a,
794 0x91b0, 0xffffffff, 0x00050004,
795 0x91b4, 0xffffffff, 0x00070006,
796 0x91b8, 0xffffffff, 0x0008000b,
797 0x91bc, 0xffffffff, 0x000a0009,
798 0x91c0, 0xffffffff, 0x000d000c,
799 0x9200, 0xffffffff, 0x00090008,
800 0x9204, 0xffffffff, 0x000b000a,
801 0x9208, 0xffffffff, 0x000c000f,
802 0x920c, 0xffffffff, 0x000e000d,
803 0x9210, 0xffffffff, 0x00110010,
804 0x9214, 0xffffffff, 0x000a0009,
805 0x9218, 0xffffffff, 0x000c000b,
806 0x921c, 0xffffffff, 0x0000000f,
807 0x9220, 0xffffffff, 0x000e000d,
808 0x9224, 0xffffffff, 0x00110010,
809 0x9228, 0xffffffff, 0x000b000a,
810 0x922c, 0xffffffff, 0x000d000c,
811 0x9230, 0xffffffff, 0x00000010,
812 0x9234, 0xffffffff, 0x000f000e,
813 0x9238, 0xffffffff, 0x00120011,
814 0x923c, 0xffffffff, 0x000c000b,
815 0x9240, 0xffffffff, 0x000e000d,
816 0x9244, 0xffffffff, 0x00000011,
817 0x9248, 0xffffffff, 0x0010000f,
818 0x924c, 0xffffffff, 0x00130012,
819 0x9250, 0xffffffff, 0x000d000c,
820 0x9254, 0xffffffff, 0x000f000e,
821 0x9258, 0xffffffff, 0x00100013,
822 0x925c, 0xffffffff, 0x00120011,
823 0x9260, 0xffffffff, 0x00150014,
824 0x9150, 0xffffffff, 0x96940200,
825 0x8708, 0xffffffff, 0x00900100,
826 0xc478, 0xffffffff, 0x00000080,
827 0xc404, 0xffffffff, 0x0020003f,
828 0x30, 0xffffffff, 0x0000001c,
829 0x34, 0x000f0000, 0x000f0000,
830 0x160c, 0xffffffff, 0x00000100,
831 0x1024, 0xffffffff, 0x00000100,
832 0x102c, 0x00000101, 0x00000000,
833 0x20a8, 0xffffffff, 0x00000104,
834 0x55e4, 0xff000fff, 0x00000100,
835 0x55e8, 0x00000001, 0x00000001,
836 0x2f50, 0x00000001, 0x00000001,
837 0x30cc, 0xc0000fff, 0x00000104,
838 0xc1e4, 0x00000001, 0x00000001,
839 0xd0c0, 0xfffffff0, 0x00000100,
840 0xd8c0, 0xfffffff0, 0x00000100
841};
842
843static const u32 verde_mgcg_cgcg_init[] =
844{
845 0xc400, 0xffffffff, 0xfffffffc,
846 0x802c, 0xffffffff, 0xe0000000,
847 0x9a60, 0xffffffff, 0x00000100,
848 0x92a4, 0xffffffff, 0x00000100,
849 0xc164, 0xffffffff, 0x00000100,
850 0x9774, 0xffffffff, 0x00000100,
851 0x8984, 0xffffffff, 0x06000100,
852 0x8a18, 0xffffffff, 0x00000100,
853 0x92a0, 0xffffffff, 0x00000100,
854 0xc380, 0xffffffff, 0x00000100,
855 0x8b28, 0xffffffff, 0x00000100,
856 0x9144, 0xffffffff, 0x00000100,
857 0x8d88, 0xffffffff, 0x00000100,
858 0x8d8c, 0xffffffff, 0x00000100,
859 0x9030, 0xffffffff, 0x00000100,
860 0x9034, 0xffffffff, 0x00000100,
861 0x9038, 0xffffffff, 0x00000100,
862 0x903c, 0xffffffff, 0x00000100,
863 0xad80, 0xffffffff, 0x00000100,
864 0xac54, 0xffffffff, 0x00000100,
865 0x897c, 0xffffffff, 0x06000100,
866 0x9868, 0xffffffff, 0x00000100,
867 0x9510, 0xffffffff, 0x00000100,
868 0xaf04, 0xffffffff, 0x00000100,
869 0xae04, 0xffffffff, 0x00000100,
870 0x949c, 0xffffffff, 0x00000100,
871 0x802c, 0xffffffff, 0xe0000000,
872 0x9160, 0xffffffff, 0x00010000,
873 0x9164, 0xffffffff, 0x00030002,
874 0x9168, 0xffffffff, 0x00040007,
875 0x916c, 0xffffffff, 0x00060005,
876 0x9170, 0xffffffff, 0x00090008,
877 0x9174, 0xffffffff, 0x00020001,
878 0x9178, 0xffffffff, 0x00040003,
879 0x917c, 0xffffffff, 0x00000007,
880 0x9180, 0xffffffff, 0x00060005,
881 0x9184, 0xffffffff, 0x00090008,
882 0x9188, 0xffffffff, 0x00030002,
883 0x918c, 0xffffffff, 0x00050004,
884 0x9190, 0xffffffff, 0x00000008,
885 0x9194, 0xffffffff, 0x00070006,
886 0x9198, 0xffffffff, 0x000a0009,
887 0x919c, 0xffffffff, 0x00040003,
888 0x91a0, 0xffffffff, 0x00060005,
889 0x91a4, 0xffffffff, 0x00000009,
890 0x91a8, 0xffffffff, 0x00080007,
891 0x91ac, 0xffffffff, 0x000b000a,
892 0x91b0, 0xffffffff, 0x00050004,
893 0x91b4, 0xffffffff, 0x00070006,
894 0x91b8, 0xffffffff, 0x0008000b,
895 0x91bc, 0xffffffff, 0x000a0009,
896 0x91c0, 0xffffffff, 0x000d000c,
897 0x9200, 0xffffffff, 0x00090008,
898 0x9204, 0xffffffff, 0x000b000a,
899 0x9208, 0xffffffff, 0x000c000f,
900 0x920c, 0xffffffff, 0x000e000d,
901 0x9210, 0xffffffff, 0x00110010,
902 0x9214, 0xffffffff, 0x000a0009,
903 0x9218, 0xffffffff, 0x000c000b,
904 0x921c, 0xffffffff, 0x0000000f,
905 0x9220, 0xffffffff, 0x000e000d,
906 0x9224, 0xffffffff, 0x00110010,
907 0x9228, 0xffffffff, 0x000b000a,
908 0x922c, 0xffffffff, 0x000d000c,
909 0x9230, 0xffffffff, 0x00000010,
910 0x9234, 0xffffffff, 0x000f000e,
911 0x9238, 0xffffffff, 0x00120011,
912 0x923c, 0xffffffff, 0x000c000b,
913 0x9240, 0xffffffff, 0x000e000d,
914 0x9244, 0xffffffff, 0x00000011,
915 0x9248, 0xffffffff, 0x0010000f,
916 0x924c, 0xffffffff, 0x00130012,
917 0x9250, 0xffffffff, 0x000d000c,
918 0x9254, 0xffffffff, 0x000f000e,
919 0x9258, 0xffffffff, 0x00100013,
920 0x925c, 0xffffffff, 0x00120011,
921 0x9260, 0xffffffff, 0x00150014,
922 0x9150, 0xffffffff, 0x96940200,
923 0x8708, 0xffffffff, 0x00900100,
924 0xc478, 0xffffffff, 0x00000080,
925 0xc404, 0xffffffff, 0x0020003f,
926 0x30, 0xffffffff, 0x0000001c,
927 0x34, 0x000f0000, 0x000f0000,
928 0x160c, 0xffffffff, 0x00000100,
929 0x1024, 0xffffffff, 0x00000100,
930 0x102c, 0x00000101, 0x00000000,
931 0x20a8, 0xffffffff, 0x00000104,
932 0x264c, 0x000c0000, 0x000c0000,
933 0x2648, 0x000c0000, 0x000c0000,
934 0x55e4, 0xff000fff, 0x00000100,
935 0x55e8, 0x00000001, 0x00000001,
936 0x2f50, 0x00000001, 0x00000001,
937 0x30cc, 0xc0000fff, 0x00000104,
938 0xc1e4, 0x00000001, 0x00000001,
939 0xd0c0, 0xfffffff0, 0x00000100,
940 0xd8c0, 0xfffffff0, 0x00000100
941};
942
943static const u32 oland_mgcg_cgcg_init[] =
944{
945 0xc400, 0xffffffff, 0xfffffffc,
946 0x802c, 0xffffffff, 0xe0000000,
947 0x9a60, 0xffffffff, 0x00000100,
948 0x92a4, 0xffffffff, 0x00000100,
949 0xc164, 0xffffffff, 0x00000100,
950 0x9774, 0xffffffff, 0x00000100,
951 0x8984, 0xffffffff, 0x06000100,
952 0x8a18, 0xffffffff, 0x00000100,
953 0x92a0, 0xffffffff, 0x00000100,
954 0xc380, 0xffffffff, 0x00000100,
955 0x8b28, 0xffffffff, 0x00000100,
956 0x9144, 0xffffffff, 0x00000100,
957 0x8d88, 0xffffffff, 0x00000100,
958 0x8d8c, 0xffffffff, 0x00000100,
959 0x9030, 0xffffffff, 0x00000100,
960 0x9034, 0xffffffff, 0x00000100,
961 0x9038, 0xffffffff, 0x00000100,
962 0x903c, 0xffffffff, 0x00000100,
963 0xad80, 0xffffffff, 0x00000100,
964 0xac54, 0xffffffff, 0x00000100,
965 0x897c, 0xffffffff, 0x06000100,
966 0x9868, 0xffffffff, 0x00000100,
967 0x9510, 0xffffffff, 0x00000100,
968 0xaf04, 0xffffffff, 0x00000100,
969 0xae04, 0xffffffff, 0x00000100,
970 0x949c, 0xffffffff, 0x00000100,
971 0x802c, 0xffffffff, 0xe0000000,
972 0x9160, 0xffffffff, 0x00010000,
973 0x9164, 0xffffffff, 0x00030002,
974 0x9168, 0xffffffff, 0x00040007,
975 0x916c, 0xffffffff, 0x00060005,
976 0x9170, 0xffffffff, 0x00090008,
977 0x9174, 0xffffffff, 0x00020001,
978 0x9178, 0xffffffff, 0x00040003,
979 0x917c, 0xffffffff, 0x00000007,
980 0x9180, 0xffffffff, 0x00060005,
981 0x9184, 0xffffffff, 0x00090008,
982 0x9188, 0xffffffff, 0x00030002,
983 0x918c, 0xffffffff, 0x00050004,
984 0x9190, 0xffffffff, 0x00000008,
985 0x9194, 0xffffffff, 0x00070006,
986 0x9198, 0xffffffff, 0x000a0009,
987 0x919c, 0xffffffff, 0x00040003,
988 0x91a0, 0xffffffff, 0x00060005,
989 0x91a4, 0xffffffff, 0x00000009,
990 0x91a8, 0xffffffff, 0x00080007,
991 0x91ac, 0xffffffff, 0x000b000a,
992 0x91b0, 0xffffffff, 0x00050004,
993 0x91b4, 0xffffffff, 0x00070006,
994 0x91b8, 0xffffffff, 0x0008000b,
995 0x91bc, 0xffffffff, 0x000a0009,
996 0x91c0, 0xffffffff, 0x000d000c,
997 0x91c4, 0xffffffff, 0x00060005,
998 0x91c8, 0xffffffff, 0x00080007,
999 0x91cc, 0xffffffff, 0x0000000b,
1000 0x91d0, 0xffffffff, 0x000a0009,
1001 0x91d4, 0xffffffff, 0x000d000c,
1002 0x9150, 0xffffffff, 0x96940200,
1003 0x8708, 0xffffffff, 0x00900100,
1004 0xc478, 0xffffffff, 0x00000080,
1005 0xc404, 0xffffffff, 0x0020003f,
1006 0x30, 0xffffffff, 0x0000001c,
1007 0x34, 0x000f0000, 0x000f0000,
1008 0x160c, 0xffffffff, 0x00000100,
1009 0x1024, 0xffffffff, 0x00000100,
1010 0x102c, 0x00000101, 0x00000000,
1011 0x20a8, 0xffffffff, 0x00000104,
1012 0x264c, 0x000c0000, 0x000c0000,
1013 0x2648, 0x000c0000, 0x000c0000,
1014 0x55e4, 0xff000fff, 0x00000100,
1015 0x55e8, 0x00000001, 0x00000001,
1016 0x2f50, 0x00000001, 0x00000001,
1017 0x30cc, 0xc0000fff, 0x00000104,
1018 0xc1e4, 0x00000001, 0x00000001,
1019 0xd0c0, 0xfffffff0, 0x00000100,
1020 0xd8c0, 0xfffffff0, 0x00000100
1021};
1022
fffbdda4
AD		 1023static const u32 hainan_mgcg_cgcg_init[] =
1024{
1025 0xc400, 0xffffffff, 0xfffffffc,
1026 0x802c, 0xffffffff, 0xe0000000,
1027 0x9a60, 0xffffffff, 0x00000100,
1028 0x92a4, 0xffffffff, 0x00000100,
1029 0xc164, 0xffffffff, 0x00000100,
1030 0x9774, 0xffffffff, 0x00000100,
1031 0x8984, 0xffffffff, 0x06000100,
1032 0x8a18, 0xffffffff, 0x00000100,
1033 0x92a0, 0xffffffff, 0x00000100,
1034 0xc380, 0xffffffff, 0x00000100,
1035 0x8b28, 0xffffffff, 0x00000100,
1036 0x9144, 0xffffffff, 0x00000100,
1037 0x8d88, 0xffffffff, 0x00000100,
1038 0x8d8c, 0xffffffff, 0x00000100,
1039 0x9030, 0xffffffff, 0x00000100,
1040 0x9034, 0xffffffff, 0x00000100,
1041 0x9038, 0xffffffff, 0x00000100,
1042 0x903c, 0xffffffff, 0x00000100,
1043 0xad80, 0xffffffff, 0x00000100,
1044 0xac54, 0xffffffff, 0x00000100,
1045 0x897c, 0xffffffff, 0x06000100,
1046 0x9868, 0xffffffff, 0x00000100,
1047 0x9510, 0xffffffff, 0x00000100,
1048 0xaf04, 0xffffffff, 0x00000100,
1049 0xae04, 0xffffffff, 0x00000100,
1050 0x949c, 0xffffffff, 0x00000100,
1051 0x802c, 0xffffffff, 0xe0000000,
1052 0x9160, 0xffffffff, 0x00010000,
1053 0x9164, 0xffffffff, 0x00030002,
1054 0x9168, 0xffffffff, 0x00040007,
1055 0x916c, 0xffffffff, 0x00060005,
1056 0x9170, 0xffffffff, 0x00090008,
1057 0x9174, 0xffffffff, 0x00020001,
1058 0x9178, 0xffffffff, 0x00040003,
1059 0x917c, 0xffffffff, 0x00000007,
1060 0x9180, 0xffffffff, 0x00060005,
1061 0x9184, 0xffffffff, 0x00090008,
1062 0x9188, 0xffffffff, 0x00030002,
1063 0x918c, 0xffffffff, 0x00050004,
1064 0x9190, 0xffffffff, 0x00000008,
1065 0x9194, 0xffffffff, 0x00070006,
1066 0x9198, 0xffffffff, 0x000a0009,
1067 0x919c, 0xffffffff, 0x00040003,
1068 0x91a0, 0xffffffff, 0x00060005,
1069 0x91a4, 0xffffffff, 0x00000009,
1070 0x91a8, 0xffffffff, 0x00080007,
1071 0x91ac, 0xffffffff, 0x000b000a,
1072 0x91b0, 0xffffffff, 0x00050004,
1073 0x91b4, 0xffffffff, 0x00070006,
1074 0x91b8, 0xffffffff, 0x0008000b,
1075 0x91bc, 0xffffffff, 0x000a0009,
1076 0x91c0, 0xffffffff, 0x000d000c,
1077 0x91c4, 0xffffffff, 0x00060005,
1078 0x91c8, 0xffffffff, 0x00080007,
1079 0x91cc, 0xffffffff, 0x0000000b,
1080 0x91d0, 0xffffffff, 0x000a0009,
1081 0x91d4, 0xffffffff, 0x000d000c,
1082 0x9150, 0xffffffff, 0x96940200,
1083 0x8708, 0xffffffff, 0x00900100,
1084 0xc478, 0xffffffff, 0x00000080,
1085 0xc404, 0xffffffff, 0x0020003f,
1086 0x30, 0xffffffff, 0x0000001c,
1087 0x34, 0x000f0000, 0x000f0000,
1088 0x160c, 0xffffffff, 0x00000100,
1089 0x1024, 0xffffffff, 0x00000100,
1090 0x20a8, 0xffffffff, 0x00000104,
1091 0x264c, 0x000c0000, 0x000c0000,
1092 0x2648, 0x000c0000, 0x000c0000,
1093 0x2f50, 0x00000001, 0x00000001,
1094 0x30cc, 0xc0000fff, 0x00000104,
1095 0xc1e4, 0x00000001, 0x00000001,
1096 0xd0c0, 0xfffffff0, 0x00000100,
1097 0xd8c0, 0xfffffff0, 0x00000100
1098};
1099
205996c0
AD		 1100static u32 verde_pg_init[] =
1101{
1102 0x353c, 0xffffffff, 0x40000,
1103 0x3538, 0xffffffff, 0x200010ff,
1104 0x353c, 0xffffffff, 0x0,
1105 0x353c, 0xffffffff, 0x0,
1106 0x353c, 0xffffffff, 0x0,
1107 0x353c, 0xffffffff, 0x0,
1108 0x353c, 0xffffffff, 0x0,
1109 0x353c, 0xffffffff, 0x7007,
1110 0x3538, 0xffffffff, 0x300010ff,
1111 0x353c, 0xffffffff, 0x0,
1112 0x353c, 0xffffffff, 0x0,
1113 0x353c, 0xffffffff, 0x0,
1114 0x353c, 0xffffffff, 0x0,
1115 0x353c, 0xffffffff, 0x0,
1116 0x353c, 0xffffffff, 0x400000,
1117 0x3538, 0xffffffff, 0x100010ff,
1118 0x353c, 0xffffffff, 0x0,
1119 0x353c, 0xffffffff, 0x0,
1120 0x353c, 0xffffffff, 0x0,
1121 0x353c, 0xffffffff, 0x0,
1122 0x353c, 0xffffffff, 0x0,
1123 0x353c, 0xffffffff, 0x120200,
1124 0x3538, 0xffffffff, 0x500010ff,
1125 0x353c, 0xffffffff, 0x0,
1126 0x353c, 0xffffffff, 0x0,
1127 0x353c, 0xffffffff, 0x0,
1128 0x353c, 0xffffffff, 0x0,
1129 0x353c, 0xffffffff, 0x0,
1130 0x353c, 0xffffffff, 0x1e1e16,
1131 0x3538, 0xffffffff, 0x600010ff,
1132 0x353c, 0xffffffff, 0x0,
1133 0x353c, 0xffffffff, 0x0,
1134 0x353c, 0xffffffff, 0x0,
1135 0x353c, 0xffffffff, 0x0,
1136 0x353c, 0xffffffff, 0x0,
1137 0x353c, 0xffffffff, 0x171f1e,
1138 0x3538, 0xffffffff, 0x700010ff,
1139 0x353c, 0xffffffff, 0x0,
1140 0x353c, 0xffffffff, 0x0,
1141 0x353c, 0xffffffff, 0x0,
1142 0x353c, 0xffffffff, 0x0,
1143 0x353c, 0xffffffff, 0x0,
1144 0x353c, 0xffffffff, 0x0,
1145 0x3538, 0xffffffff, 0x9ff,
1146 0x3500, 0xffffffff, 0x0,
1147 0x3504, 0xffffffff, 0x10000800,
1148 0x3504, 0xffffffff, 0xf,
1149 0x3504, 0xffffffff, 0xf,
1150 0x3500, 0xffffffff, 0x4,
1151 0x3504, 0xffffffff, 0x1000051e,
1152 0x3504, 0xffffffff, 0xffff,
1153 0x3504, 0xffffffff, 0xffff,
1154 0x3500, 0xffffffff, 0x8,
1155 0x3504, 0xffffffff, 0x80500,
1156 0x3500, 0xffffffff, 0x12,
1157 0x3504, 0xffffffff, 0x9050c,
1158 0x3500, 0xffffffff, 0x1d,
1159 0x3504, 0xffffffff, 0xb052c,
1160 0x3500, 0xffffffff, 0x2a,
1161 0x3504, 0xffffffff, 0x1053e,
1162 0x3500, 0xffffffff, 0x2d,
1163 0x3504, 0xffffffff, 0x10546,
1164 0x3500, 0xffffffff, 0x30,
1165 0x3504, 0xffffffff, 0xa054e,
1166 0x3500, 0xffffffff, 0x3c,
1167 0x3504, 0xffffffff, 0x1055f,
1168 0x3500, 0xffffffff, 0x3f,
1169 0x3504, 0xffffffff, 0x10567,
1170 0x3500, 0xffffffff, 0x42,
1171 0x3504, 0xffffffff, 0x1056f,
1172 0x3500, 0xffffffff, 0x45,
1173 0x3504, 0xffffffff, 0x10572,
1174 0x3500, 0xffffffff, 0x48,
1175 0x3504, 0xffffffff, 0x20575,
1176 0x3500, 0xffffffff, 0x4c,
1177 0x3504, 0xffffffff, 0x190801,
1178 0x3500, 0xffffffff, 0x67,
1179 0x3504, 0xffffffff, 0x1082a,
1180 0x3500, 0xffffffff, 0x6a,
1181 0x3504, 0xffffffff, 0x1b082d,
1182 0x3500, 0xffffffff, 0x87,
1183 0x3504, 0xffffffff, 0x310851,
1184 0x3500, 0xffffffff, 0xba,
1185 0x3504, 0xffffffff, 0x891,
1186 0x3500, 0xffffffff, 0xbc,
1187 0x3504, 0xffffffff, 0x893,
1188 0x3500, 0xffffffff, 0xbe,
1189 0x3504, 0xffffffff, 0x20895,
1190 0x3500, 0xffffffff, 0xc2,
1191 0x3504, 0xffffffff, 0x20899,
1192 0x3500, 0xffffffff, 0xc6,
1193 0x3504, 0xffffffff, 0x2089d,
1194 0x3500, 0xffffffff, 0xca,
1195 0x3504, 0xffffffff, 0x8a1,
1196 0x3500, 0xffffffff, 0xcc,
1197 0x3504, 0xffffffff, 0x8a3,
1198 0x3500, 0xffffffff, 0xce,
1199 0x3504, 0xffffffff, 0x308a5,
1200 0x3500, 0xffffffff, 0xd3,
1201 0x3504, 0xffffffff, 0x6d08cd,
1202 0x3500, 0xffffffff, 0x142,
1203 0x3504, 0xffffffff, 0x2000095a,
1204 0x3504, 0xffffffff, 0x1,
1205 0x3500, 0xffffffff, 0x144,
1206 0x3504, 0xffffffff, 0x301f095b,
1207 0x3500, 0xffffffff, 0x165,
1208 0x3504, 0xffffffff, 0xc094d,
1209 0x3500, 0xffffffff, 0x173,
1210 0x3504, 0xffffffff, 0xf096d,
1211 0x3500, 0xffffffff, 0x184,
1212 0x3504, 0xffffffff, 0x15097f,
1213 0x3500, 0xffffffff, 0x19b,
1214 0x3504, 0xffffffff, 0xc0998,
1215 0x3500, 0xffffffff, 0x1a9,
1216 0x3504, 0xffffffff, 0x409a7,
1217 0x3500, 0xffffffff, 0x1af,
1218 0x3504, 0xffffffff, 0xcdc,
1219 0x3500, 0xffffffff, 0x1b1,
1220 0x3504, 0xffffffff, 0x800,
1221 0x3508, 0xffffffff, 0x6c9b2000,
1222 0x3510, 0xfc00, 0x2000,
1223 0x3544, 0xffffffff, 0xfc0,
1224 0x28d4, 0x00000100, 0x100
1225};
1226
1227static void si_init_golden_registers(struct radeon_device *rdev)
1228{
1229 switch (rdev->family) {
1230 case CHIP_TAHITI:
1231 radeon_program_register_sequence(rdev,
1232 tahiti_golden_registers,
1233 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1234 radeon_program_register_sequence(rdev,
1235 tahiti_golden_rlc_registers,
1236 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1237 radeon_program_register_sequence(rdev,
1238 tahiti_mgcg_cgcg_init,
1239 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1240 radeon_program_register_sequence(rdev,
1241 tahiti_golden_registers2,
1242 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1243 break;
1244 case CHIP_PITCAIRN:
1245 radeon_program_register_sequence(rdev,
1246 pitcairn_golden_registers,
1247 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1248 radeon_program_register_sequence(rdev,
1249 pitcairn_golden_rlc_registers,
1250 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1251 radeon_program_register_sequence(rdev,
1252 pitcairn_mgcg_cgcg_init,
1253 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1254 break;
1255 case CHIP_VERDE:
1256 radeon_program_register_sequence(rdev,
1257 verde_golden_registers,
1258 (const u32)ARRAY_SIZE(verde_golden_registers));
1259 radeon_program_register_sequence(rdev,
1260 verde_golden_rlc_registers,
1261 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1262 radeon_program_register_sequence(rdev,
1263 verde_mgcg_cgcg_init,
1264 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1265 radeon_program_register_sequence(rdev,
1266 verde_pg_init,
1267 (const u32)ARRAY_SIZE(verde_pg_init));
1268 break;
1269 case CHIP_OLAND:
1270 radeon_program_register_sequence(rdev,
1271 oland_golden_registers,
1272 (const u32)ARRAY_SIZE(oland_golden_registers));
1273 radeon_program_register_sequence(rdev,
1274 oland_golden_rlc_registers,
1275 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1276 radeon_program_register_sequence(rdev,
1277 oland_mgcg_cgcg_init,
1278 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1279 break;
fffbdda4
AD		 1280 case CHIP_HAINAN:
1281 radeon_program_register_sequence(rdev,
1282 hainan_golden_registers,
1283 (const u32)ARRAY_SIZE(hainan_golden_registers));
1284 radeon_program_register_sequence(rdev,
1285 hainan_golden_registers2,
1286 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1287 radeon_program_register_sequence(rdev,
1288 hainan_mgcg_cgcg_init,
1289 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1290 break;
205996c0
AD		 1291 default:
1292 break;
1293 }
1294}
1295
4af692f6
AD		 1296/**
1297 * si_get_allowed_info_register - fetch the register for the info ioctl
1298 *
1299 * @rdev: radeon_device pointer
1300 * @reg: register offset in bytes
1301 * @val: register value
1302 *
1303 * Returns 0 for success or -EINVAL for an invalid register
1304 *
1305 */
1306int si_get_allowed_info_register(struct radeon_device *rdev,
1307 u32 reg, u32 *val)
1308{
1309 switch (reg) {
1310 case GRBM_STATUS:
1311 case GRBM_STATUS2:
1312 case GRBM_STATUS_SE0:
1313 case GRBM_STATUS_SE1:
1314 case SRBM_STATUS:
1315 case SRBM_STATUS2:
1316 case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1317 case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1318 case UVD_STATUS:
1319 *val = RREG32(reg);
1320 return 0;
1321 default:
1322 return -EINVAL;
1323 }
1324}
1325
454d2e2a
AD		 1326#define PCIE_BUS_CLK 10000
1327#define TCLK (PCIE_BUS_CLK / 10)
1328
1329/**
1330 * si_get_xclk - get the xclk
1331 *
1332 * @rdev: radeon_device pointer
1333 *
1334 * Returns the reference clock used by the gfx engine
1335 * (SI).
1336 */
1337u32 si_get_xclk(struct radeon_device *rdev)
1338{
3cf8bb1a 1339 u32 reference_clock = rdev->clock.spll.reference_freq;
454d2e2a
AD		 1340 u32 tmp;
1341
1342 tmp = RREG32(CG_CLKPIN_CNTL_2);
1343 if (tmp & MUX_TCLK_TO_XCLK)
1344 return TCLK;
1345
1346 tmp = RREG32(CG_CLKPIN_CNTL);
1347 if (tmp & XTALIN_DIVIDE)
1348 return reference_clock / 4;
1349
1350 return reference_clock;
1351}
1352
1bd47d2e
AD		 1353/* get temperature in millidegrees */
1354int si_get_temp(struct radeon_device *rdev)
1355{
1356 u32 temp;
1357 int actual_temp = 0;
1358
1359 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1360 CTF_TEMP_SHIFT;
1361
1362 if (temp & 0x200)
1363 actual_temp = 255;
1364 else
1365 actual_temp = temp & 0x1ff;
1366
1367 actual_temp = (actual_temp * 1000);
1368
1369 return actual_temp;
1370}
1371
8b074dd6
AD		 1372#define TAHITI_IO_MC_REGS_SIZE 36
1373
1374static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1375 {0x0000006f, 0x03044000},
1376 {0x00000070, 0x0480c018},
1377 {0x00000071, 0x00000040},
1378 {0x00000072, 0x01000000},
1379 {0x00000074, 0x000000ff},
1380 {0x00000075, 0x00143400},
1381 {0x00000076, 0x08ec0800},
1382 {0x00000077, 0x040000cc},
1383 {0x00000079, 0x00000000},
1384 {0x0000007a, 0x21000409},
1385 {0x0000007c, 0x00000000},
1386 {0x0000007d, 0xe8000000},
1387 {0x0000007e, 0x044408a8},
1388 {0x0000007f, 0x00000003},
1389 {0x00000080, 0x00000000},
1390 {0x00000081, 0x01000000},
1391 {0x00000082, 0x02000000},
1392 {0x00000083, 0x00000000},
1393 {0x00000084, 0xe3f3e4f4},
1394 {0x00000085, 0x00052024},
1395 {0x00000087, 0x00000000},
1396 {0x00000088, 0x66036603},
1397 {0x00000089, 0x01000000},
1398 {0x0000008b, 0x1c0a0000},
1399 {0x0000008c, 0xff010000},
1400 {0x0000008e, 0xffffefff},
1401 {0x0000008f, 0xfff3efff},
1402 {0x00000090, 0xfff3efbf},
1403 {0x00000094, 0x00101101},
1404 {0x00000095, 0x00000fff},
1405 {0x00000096, 0x00116fff},
1406 {0x00000097, 0x60010000},
1407 {0x00000098, 0x10010000},
1408 {0x00000099, 0x00006000},
1409 {0x0000009a, 0x00001000},
1410 {0x0000009f, 0x00a77400}
1411};
1412
1413static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1414 {0x0000006f, 0x03044000},
1415 {0x00000070, 0x0480c018},
1416 {0x00000071, 0x00000040},
1417 {0x00000072, 0x01000000},
1418 {0x00000074, 0x000000ff},
1419 {0x00000075, 0x00143400},
1420 {0x00000076, 0x08ec0800},
1421 {0x00000077, 0x040000cc},
1422 {0x00000079, 0x00000000},
1423 {0x0000007a, 0x21000409},
1424 {0x0000007c, 0x00000000},
1425 {0x0000007d, 0xe8000000},
1426 {0x0000007e, 0x044408a8},
1427 {0x0000007f, 0x00000003},
1428 {0x00000080, 0x00000000},
1429 {0x00000081, 0x01000000},
1430 {0x00000082, 0x02000000},
1431 {0x00000083, 0x00000000},
1432 {0x00000084, 0xe3f3e4f4},
1433 {0x00000085, 0x00052024},
1434 {0x00000087, 0x00000000},
1435 {0x00000088, 0x66036603},
1436 {0x00000089, 0x01000000},
1437 {0x0000008b, 0x1c0a0000},
1438 {0x0000008c, 0xff010000},
1439 {0x0000008e, 0xffffefff},
1440 {0x0000008f, 0xfff3efff},
1441 {0x00000090, 0xfff3efbf},
1442 {0x00000094, 0x00101101},
1443 {0x00000095, 0x00000fff},
1444 {0x00000096, 0x00116fff},
1445 {0x00000097, 0x60010000},
1446 {0x00000098, 0x10010000},
1447 {0x00000099, 0x00006000},
1448 {0x0000009a, 0x00001000},
1449 {0x0000009f, 0x00a47400}
1450};
1451
1452static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1453 {0x0000006f, 0x03044000},
1454 {0x00000070, 0x0480c018},
1455 {0x00000071, 0x00000040},
1456 {0x00000072, 0x01000000},
1457 {0x00000074, 0x000000ff},
1458 {0x00000075, 0x00143400},
1459 {0x00000076, 0x08ec0800},
1460 {0x00000077, 0x040000cc},
1461 {0x00000079, 0x00000000},
1462 {0x0000007a, 0x21000409},
1463 {0x0000007c, 0x00000000},
1464 {0x0000007d, 0xe8000000},
1465 {0x0000007e, 0x044408a8},
1466 {0x0000007f, 0x00000003},
1467 {0x00000080, 0x00000000},
1468 {0x00000081, 0x01000000},
1469 {0x00000082, 0x02000000},
1470 {0x00000083, 0x00000000},
1471 {0x00000084, 0xe3f3e4f4},
1472 {0x00000085, 0x00052024},
1473 {0x00000087, 0x00000000},
1474 {0x00000088, 0x66036603},
1475 {0x00000089, 0x01000000},
1476 {0x0000008b, 0x1c0a0000},
1477 {0x0000008c, 0xff010000},
1478 {0x0000008e, 0xffffefff},
1479 {0x0000008f, 0xfff3efff},
1480 {0x00000090, 0xfff3efbf},
1481 {0x00000094, 0x00101101},
1482 {0x00000095, 0x00000fff},
1483 {0x00000096, 0x00116fff},
1484 {0x00000097, 0x60010000},
1485 {0x00000098, 0x10010000},
1486 {0x00000099, 0x00006000},
1487 {0x0000009a, 0x00001000},
1488 {0x0000009f, 0x00a37400}
1489};
1490
bcc7f5d2
AD		 1491static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1492 {0x0000006f, 0x03044000},
1493 {0x00000070, 0x0480c018},
1494 {0x00000071, 0x00000040},
1495 {0x00000072, 0x01000000},
1496 {0x00000074, 0x000000ff},
1497 {0x00000075, 0x00143400},
1498 {0x00000076, 0x08ec0800},
1499 {0x00000077, 0x040000cc},
1500 {0x00000079, 0x00000000},
1501 {0x0000007a, 0x21000409},
1502 {0x0000007c, 0x00000000},
1503 {0x0000007d, 0xe8000000},
1504 {0x0000007e, 0x044408a8},
1505 {0x0000007f, 0x00000003},
1506 {0x00000080, 0x00000000},
1507 {0x00000081, 0x01000000},
1508 {0x00000082, 0x02000000},
1509 {0x00000083, 0x00000000},
1510 {0x00000084, 0xe3f3e4f4},
1511 {0x00000085, 0x00052024},
1512 {0x00000087, 0x00000000},
1513 {0x00000088, 0x66036603},
1514 {0x00000089, 0x01000000},
1515 {0x0000008b, 0x1c0a0000},
1516 {0x0000008c, 0xff010000},
1517 {0x0000008e, 0xffffefff},
1518 {0x0000008f, 0xfff3efff},
1519 {0x00000090, 0xfff3efbf},
1520 {0x00000094, 0x00101101},
1521 {0x00000095, 0x00000fff},
1522 {0x00000096, 0x00116fff},
1523 {0x00000097, 0x60010000},
1524 {0x00000098, 0x10010000},
1525 {0x00000099, 0x00006000},
1526 {0x0000009a, 0x00001000},
1527 {0x0000009f, 0x00a17730}
1528};
1529
c04c00b4
AD		 1530static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1531 {0x0000006f, 0x03044000},
1532 {0x00000070, 0x0480c018},
1533 {0x00000071, 0x00000040},
1534 {0x00000072, 0x01000000},
1535 {0x00000074, 0x000000ff},
1536 {0x00000075, 0x00143400},
1537 {0x00000076, 0x08ec0800},
1538 {0x00000077, 0x040000cc},
1539 {0x00000079, 0x00000000},
1540 {0x0000007a, 0x21000409},
1541 {0x0000007c, 0x00000000},
1542 {0x0000007d, 0xe8000000},
1543 {0x0000007e, 0x044408a8},
1544 {0x0000007f, 0x00000003},
1545 {0x00000080, 0x00000000},
1546 {0x00000081, 0x01000000},
1547 {0x00000082, 0x02000000},
1548 {0x00000083, 0x00000000},
1549 {0x00000084, 0xe3f3e4f4},
1550 {0x00000085, 0x00052024},
1551 {0x00000087, 0x00000000},
1552 {0x00000088, 0x66036603},
1553 {0x00000089, 0x01000000},
1554 {0x0000008b, 0x1c0a0000},
1555 {0x0000008c, 0xff010000},
1556 {0x0000008e, 0xffffefff},
1557 {0x0000008f, 0xfff3efff},
1558 {0x00000090, 0xfff3efbf},
1559 {0x00000094, 0x00101101},
1560 {0x00000095, 0x00000fff},
1561 {0x00000096, 0x00116fff},
1562 {0x00000097, 0x60010000},
1563 {0x00000098, 0x10010000},
1564 {0x00000099, 0x00006000},
1565 {0x0000009a, 0x00001000},
1566 {0x0000009f, 0x00a07730}
1567};
1568
8b074dd6 1569/* ucode loading */
6c7bccea 1570int si_mc_load_microcode(struct radeon_device *rdev)
8b074dd6 1571{
629bd33c
AD		 1572 const __be32 *fw_data = NULL;
1573 const __le32 *new_fw_data = NULL;
ddbbd3be 1574 u32 running;
629bd33c
AD		 1575 u32 *io_mc_regs = NULL;
1576 const __le32 *new_io_mc_regs = NULL;
8c79bae6 1577 int i, regs_size, ucode_size;
8b074dd6
AD		 1578
1579 if (!rdev->mc_fw)
1580 return -EINVAL;
1581
629bd33c
AD		 1582 if (rdev->new_fw) {
1583 const struct mc_firmware_header_v1_0 *hdr =
1584 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1585
1586 radeon_ucode_print_mc_hdr(&hdr->header);
1587 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1588 new_io_mc_regs = (const __le32 *)
1589 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1590 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1591 new_fw_data = (const __le32 *)
1592 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1593 } else {
1594 ucode_size = rdev->mc_fw->size / 4;
8c79bae6 1595
629bd33c
AD		 1596 switch (rdev->family) {
1597 case CHIP_TAHITI:
1598 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1599 regs_size = TAHITI_IO_MC_REGS_SIZE;
1600 break;
1601 case CHIP_PITCAIRN:
1602 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1603 regs_size = TAHITI_IO_MC_REGS_SIZE;
1604 break;
1605 case CHIP_VERDE:
1606 default:
1607 io_mc_regs = (u32 *)&verde_io_mc_regs;
1608 regs_size = TAHITI_IO_MC_REGS_SIZE;
1609 break;
1610 case CHIP_OLAND:
1611 io_mc_regs = (u32 *)&oland_io_mc_regs;
1612 regs_size = TAHITI_IO_MC_REGS_SIZE;
1613 break;
1614 case CHIP_HAINAN:
1615 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1616 regs_size = TAHITI_IO_MC_REGS_SIZE;
1617 break;
1618 }
1619 fw_data = (const __be32 *)rdev->mc_fw->data;
8b074dd6
AD		 1620 }
1621
1622 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1623
1624 if (running == 0) {
8b074dd6
AD		 1625 /* reset the engine and set to writable */
1626 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1627 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1628
1629 /* load mc io regs */
1630 for (i = 0; i < regs_size; i++) {
629bd33c
AD		 1631 if (rdev->new_fw) {
1632 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1633 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1634 } else {
1635 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1636 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1637 }
8b074dd6
AD		 1638 }
1639 /* load the MC ucode */
629bd33c
AD		 1640 for (i = 0; i < ucode_size; i++) {
1641 if (rdev->new_fw)
1642 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1643 else
1644 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1645 }
8b074dd6
AD		 1646
1647 /* put the engine back into the active state */
1648 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1649 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1650 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1651
1652 /* wait for training to complete */
1653 for (i = 0; i < rdev->usec_timeout; i++) {
1654 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1655 break;
1656 udelay(1);
1657 }
1658 for (i = 0; i < rdev->usec_timeout; i++) {
1659 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1660 break;
1661 udelay(1);
1662 }
8b074dd6
AD		 1663 }
1664
1665 return 0;
1666}
1667
0f0de06c
AD		 1668static int si_init_microcode(struct radeon_device *rdev)
1669{
0f0de06c 1670 const char *chip_name;
629bd33c 1671 const char *new_chip_name;
0f0de06c 1672 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1ebe9280 1673 size_t smc_req_size, mc2_req_size;
0f0de06c
AD		 1674 char fw_name[30];
1675 int err;
629bd33c 1676 int new_fw = 0;
861c7fde 1677 bool new_smc = false;
ef736d39 1678 bool si58_fw = false;
4e6e98b1 1679 bool banks2_fw = false;
0f0de06c
AD		 1680
1681 DRM_DEBUG("\n");
1682
0f0de06c
AD		 1683 switch (rdev->family) {
1684 case CHIP_TAHITI:
1685 chip_name = "TAHITI";
629bd33c 1686 new_chip_name = "tahiti";
0f0de06c
AD		 1687 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1688 me_req_size = SI_PM4_UCODE_SIZE * 4;
1689 ce_req_size = SI_CE_UCODE_SIZE * 4;
1690 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1691 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1692 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
a9e61410 1693 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1694 break;
1695 case CHIP_PITCAIRN:
1696 chip_name = "PITCAIRN";
6458bd4d
AD		 1697 if ((rdev->pdev->revision == 0x81) &&
1698 ((rdev->pdev->device == 0x6810) ||
1699 (rdev->pdev->device == 0x6811)))
861c7fde 1700 new_smc = true;
629bd33c 1701 new_chip_name = "pitcairn";
0f0de06c
AD		 1702 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1703 me_req_size = SI_PM4_UCODE_SIZE * 4;
1704 ce_req_size = SI_CE_UCODE_SIZE * 4;
1705 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1706 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1707 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
a9e61410 1708 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1709 break;
1710 case CHIP_VERDE:
1711 chip_name = "VERDE";
6458bd4d
AD		 1712 if (((rdev->pdev->device == 0x6820) &&
1713 ((rdev->pdev->revision == 0x81) ||
1714 (rdev->pdev->revision == 0x83))) ||
1715 ((rdev->pdev->device == 0x6821) &&
1716 ((rdev->pdev->revision == 0x83) ||
1717 (rdev->pdev->revision == 0x87))) ||
1718 ((rdev->pdev->revision == 0x87) &&
1719 ((rdev->pdev->device == 0x6823) ||
1720 (rdev->pdev->device == 0x682b))))
861c7fde 1721 new_smc = true;
629bd33c 1722 new_chip_name = "verde";
0f0de06c
AD		 1723 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1724 me_req_size = SI_PM4_UCODE_SIZE * 4;
1725 ce_req_size = SI_CE_UCODE_SIZE * 4;
1726 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1727 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1728 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
a9e61410 1729 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
0f0de06c 1730 break;
bcc7f5d2
AD		 1731 case CHIP_OLAND:
1732 chip_name = "OLAND";
6458bd4d
AD		 1733 if (((rdev->pdev->revision == 0x81) &&
1734 ((rdev->pdev->device == 0x6600) ||
1735 (rdev->pdev->device == 0x6604) ||
1736 (rdev->pdev->device == 0x6605) ||
1737 (rdev->pdev->device == 0x6610))) ||
1738 ((rdev->pdev->revision == 0x83) &&
1739 (rdev->pdev->device == 0x6610)))
861c7fde 1740 new_smc = true;
629bd33c 1741 new_chip_name = "oland";
bcc7f5d2
AD		 1742 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1743 me_req_size = SI_PM4_UCODE_SIZE * 4;
1744 ce_req_size = SI_CE_UCODE_SIZE * 4;
1745 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1ebe9280 1746 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1747 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
bcc7f5d2 1748 break;
c04c00b4
AD		 1749 case CHIP_HAINAN:
1750 chip_name = "HAINAN";
6458bd4d
AD		 1751 if (((rdev->pdev->revision == 0x81) &&
1752 (rdev->pdev->device == 0x6660)) ||
1753 ((rdev->pdev->revision == 0x83) &&
1754 ((rdev->pdev->device == 0x6660) ||
1755 (rdev->pdev->device == 0x6663) ||
1756 (rdev->pdev->device == 0x6665) ||
4e6e98b1 1757 (rdev->pdev->device == 0x6667))))
861c7fde 1758 new_smc = true;
4e6e98b1
AD		 1759 else if ((rdev->pdev->revision == 0xc3) &&
1760 (rdev->pdev->device == 0x6665))
1761 banks2_fw = true;
629bd33c 1762 new_chip_name = "hainan";
c04c00b4
AD		 1763 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1764 me_req_size = SI_PM4_UCODE_SIZE * 4;
1765 ce_req_size = SI_CE_UCODE_SIZE * 4;
1766 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1ebe9280 1767 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1768 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
c04c00b4 1769 break;
0f0de06c
AD		 1770 default: BUG();
1771 }
1772
ef736d39
AD		 1773 /* this memory configuration requires special firmware */
1774 if (((RREG32(MC_SEQ_MISC0) & 0xff000000) >> 24) == 0x58)
1775 si58_fw = true;
1776
629bd33c 1777 DRM_INFO("Loading %s Microcode\n", new_chip_name);
0f0de06c 1778
629bd33c 1779 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 1780 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
629bd33c
AD		 1781 if (err) {
1782 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1783 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1784 if (err)
1785 goto out;
1786 if (rdev->pfp_fw->size != pfp_req_size) {
7ca85295 1787 pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1788 rdev->pfp_fw->size, fw_name);
1789 err = -EINVAL;
1790 goto out;
1791 }
1792 } else {
1793 err = radeon_ucode_validate(rdev->pfp_fw);
1794 if (err) {
7ca85295 1795 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c
AD		 1796 fw_name);
1797 goto out;
1798 } else {
1799 new_fw++;
1800 }
0f0de06c
AD		 1801 }
1802
629bd33c 1803 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 1804 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
629bd33c
AD		 1805 if (err) {
1806 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1807 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1808 if (err)
1809 goto out;
1810 if (rdev->me_fw->size != me_req_size) {
7ca85295 1811 pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1812 rdev->me_fw->size, fw_name);
1813 err = -EINVAL;
1814 }
1815 } else {
1816 err = radeon_ucode_validate(rdev->me_fw);
1817 if (err) {
7ca85295 1818 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c
AD		 1819 fw_name);
1820 goto out;
1821 } else {
1822 new_fw++;
1823 }
0f0de06c
AD		 1824 }
1825
629bd33c 1826 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 1827 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
629bd33c
AD		 1828 if (err) {
1829 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1830 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1831 if (err)
1832 goto out;
1833 if (rdev->ce_fw->size != ce_req_size) {
7ca85295 1834 pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1835 rdev->ce_fw->size, fw_name);
1836 err = -EINVAL;
1837 }
1838 } else {
1839 err = radeon_ucode_validate(rdev->ce_fw);
1840 if (err) {
7ca85295 1841 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c
AD		 1842 fw_name);
1843 goto out;
1844 } else {
1845 new_fw++;
1846 }
0f0de06c
AD		 1847 }
1848
629bd33c 1849 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 1850 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
629bd33c
AD		 1851 if (err) {
1852 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1853 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1854 if (err)
1855 goto out;
1856 if (rdev->rlc_fw->size != rlc_req_size) {
7ca85295 1857 pr_err("si_rlc: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1858 rdev->rlc_fw->size, fw_name);
1859 err = -EINVAL;
1860 }
1861 } else {
1862 err = radeon_ucode_validate(rdev->rlc_fw);
1863 if (err) {
7ca85295 1864 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c
AD		 1865 fw_name);
1866 goto out;
1867 } else {
1868 new_fw++;
1869 }
0f0de06c
AD		 1870 }
1871
ef736d39
AD		 1872 if (si58_fw)
1873 snprintf(fw_name, sizeof(fw_name), "radeon/si58_mc.bin");
1874 else
1875 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 1876 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1ebe9280 1877 if (err) {
629bd33c 1878 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1ebe9280 1879 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
629bd33c
AD		 1880 if (err) {
1881 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1882 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1883 if (err)
1884 goto out;
1885 }
1886 if ((rdev->mc_fw->size != mc_req_size) &&
1887 (rdev->mc_fw->size != mc2_req_size)) {
7ca85295 1888 pr_err("si_mc: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1889 rdev->mc_fw->size, fw_name);
1890 err = -EINVAL;
1891 }
1892 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1893 } else {
1894 err = radeon_ucode_validate(rdev->mc_fw);
1895 if (err) {
7ca85295 1896 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c 1897 fw_name);
1ebe9280 1898 goto out;
629bd33c
AD		 1899 } else {
1900 new_fw++;
1901 }
1ebe9280 1902 }
0f0de06c 1903
4e6e98b1
AD		 1904 if (banks2_fw)
1905 snprintf(fw_name, sizeof(fw_name), "radeon/banks_k_2_smc.bin");
1906 else if (new_smc)
861c7fde
AD		 1907 snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
1908 else
1909 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
0a168933 1910 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
8a53fa23 1911 if (err) {
629bd33c
AD		 1912 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1913 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1914 if (err) {
7ca85295 1915 pr_err("smc: error loading firmware \"%s\"\n", fw_name);
629bd33c
AD		 1916 release_firmware(rdev->smc_fw);
1917 rdev->smc_fw = NULL;
1918 err = 0;
1919 } else if (rdev->smc_fw->size != smc_req_size) {
7ca85295 1920 pr_err("si_smc: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1921 rdev->smc_fw->size, fw_name);
1922 err = -EINVAL;
1923 }
1924 } else {
1925 err = radeon_ucode_validate(rdev->smc_fw);
1926 if (err) {
7ca85295 1927 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c
AD		 1928 fw_name);
1929 goto out;
1930 } else {
1931 new_fw++;
1932 }
a9e61410
AD		 1933 }
1934
629bd33c
AD		 1935 if (new_fw == 0) {
1936 rdev->new_fw = false;
1937 } else if (new_fw < 6) {
7ca85295 1938 pr_err("si_fw: mixing new and old firmware!\n");
629bd33c
AD		 1939 err = -EINVAL;
1940 } else {
1941 rdev->new_fw = true;
1942 }
0f0de06c 1943out:
0f0de06c
AD		 1944 if (err) {
1945 if (err != -EINVAL)
7ca85295 1946 pr_err("si_cp: Failed to load firmware \"%s\"\n",
0f0de06c
AD		 1947 fw_name);
1948 release_firmware(rdev->pfp_fw);
1949 rdev->pfp_fw = NULL;
1950 release_firmware(rdev->me_fw);
1951 rdev->me_fw = NULL;
1952 release_firmware(rdev->ce_fw);
1953 rdev->ce_fw = NULL;
1954 release_firmware(rdev->rlc_fw);
1955 rdev->rlc_fw = NULL;
1956 release_firmware(rdev->mc_fw);
1957 rdev->mc_fw = NULL;
a9e61410
AD		 1958 release_firmware(rdev->smc_fw);
1959 rdev->smc_fw = NULL;
0f0de06c
AD		 1960 }
1961 return err;
1962}
1963
43b3cd99
AD		 1964/* watermark setup */
1965static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1966 struct radeon_crtc *radeon_crtc,
1967 struct drm_display_mode *mode,
1968 struct drm_display_mode *other_mode)
1969{
290d2457
AD		 1970 u32 tmp, buffer_alloc, i;
1971 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
43b3cd99
AD		 1972 /*
1973 * Line Buffer Setup
1974 * There are 3 line buffers, each one shared by 2 display controllers.
1975 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1976 * the display controllers. The paritioning is done via one of four
1977 * preset allocations specified in bits 21:20:
1978 * 0 - half lb
1979 * 2 - whole lb, other crtc must be disabled
1980 */
1981 /* this can get tricky if we have two large displays on a paired group
1982 * of crtcs. Ideally for multiple large displays we'd assign them to
1983 * non-linked crtcs for maximum line buffer allocation.
1984 */
1985 if (radeon_crtc->base.enabled && mode) {
290d2457 1986 if (other_mode) {
43b3cd99 1987 tmp = 0; /* 1/2 */
290d2457
AD		 1988 buffer_alloc = 1;
1989 } else {
43b3cd99 1990 tmp = 2; /* whole */
290d2457
AD		 1991 buffer_alloc = 2;
1992 }
1993 } else {
43b3cd99 1994 tmp = 0;
290d2457
AD		 1995 buffer_alloc = 0;
1996 }
43b3cd99
AD		 1997
1998 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1999 DC_LB_MEMORY_CONFIG(tmp));
2000
290d2457
AD		 2001 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
2002 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
2003 for (i = 0; i < rdev->usec_timeout; i++) {
2004 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
2005 DMIF_BUFFERS_ALLOCATED_COMPLETED)
2006 break;
2007 udelay(1);
2008 }
2009
43b3cd99
AD		 2010 if (radeon_crtc->base.enabled && mode) {
2011 switch (tmp) {
2012 case 0:
2013 default:
2014 return 4096 * 2;
2015 case 2:
2016 return 8192 * 2;
2017 }
2018 }
2019
2020 /* controller not enabled, so no lb used */
2021 return 0;
2022}
2023
ca7db22b 2024static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 2025{
2026 u32 tmp = RREG32(MC_SHARED_CHMAP);
2027
2028 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2029 case 0:
2030 default:
2031 return 1;
2032 case 1:
2033 return 2;
2034 case 2:
2035 return 4;
2036 case 3:
2037 return 8;
2038 case 4:
2039 return 3;
2040 case 5:
2041 return 6;
2042 case 6:
2043 return 10;
2044 case 7:
2045 return 12;
2046 case 8:
2047 return 16;
2048 }
2049}
2050
2051struct dce6_wm_params {
2052 u32 dram_channels; /* number of dram channels */
2053 u32 yclk; /* bandwidth per dram data pin in kHz */
2054 u32 sclk; /* engine clock in kHz */
2055 u32 disp_clk; /* display clock in kHz */
2056 u32 src_width; /* viewport width */
2057 u32 active_time; /* active display time in ns */
2058 u32 blank_time; /* blank time in ns */
2059 bool interlaced; /* mode is interlaced */
2060 fixed20_12 vsc; /* vertical scale ratio */
2061 u32 num_heads; /* number of active crtcs */
2062 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2063 u32 lb_size; /* line buffer allocated to pipe */
2064 u32 vtaps; /* vertical scaler taps */
2065};
2066
2067static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
2068{
2069 /* Calculate raw DRAM Bandwidth */
2070 fixed20_12 dram_efficiency; /* 0.7 */
2071 fixed20_12 yclk, dram_channels, bandwidth;
2072 fixed20_12 a;
2073
2074 a.full = dfixed_const(1000);
2075 yclk.full = dfixed_const(wm->yclk);
2076 yclk.full = dfixed_div(yclk, a);
2077 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2078 a.full = dfixed_const(10);
2079 dram_efficiency.full = dfixed_const(7);
2080 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2081 bandwidth.full = dfixed_mul(dram_channels, yclk);
2082 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2083
2084 return dfixed_trunc(bandwidth);
2085}
2086
2087static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2088{
2089 /* Calculate DRAM Bandwidth and the part allocated to display. */
2090 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2091 fixed20_12 yclk, dram_channels, bandwidth;
2092 fixed20_12 a;
2093
2094 a.full = dfixed_const(1000);
2095 yclk.full = dfixed_const(wm->yclk);
2096 yclk.full = dfixed_div(yclk, a);
2097 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2098 a.full = dfixed_const(10);
2099 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2100 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2101 bandwidth.full = dfixed_mul(dram_channels, yclk);
2102 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2103
2104 return dfixed_trunc(bandwidth);
2105}
2106
2107static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2108{
2109 /* Calculate the display Data return Bandwidth */
2110 fixed20_12 return_efficiency; /* 0.8 */
2111 fixed20_12 sclk, bandwidth;
2112 fixed20_12 a;
2113
2114 a.full = dfixed_const(1000);
2115 sclk.full = dfixed_const(wm->sclk);
2116 sclk.full = dfixed_div(sclk, a);
2117 a.full = dfixed_const(10);
2118 return_efficiency.full = dfixed_const(8);
2119 return_efficiency.full = dfixed_div(return_efficiency, a);
2120 a.full = dfixed_const(32);
2121 bandwidth.full = dfixed_mul(a, sclk);
2122 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2123
2124 return dfixed_trunc(bandwidth);
2125}
2126
2127static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2128{
2129 return 32;
2130}
2131
2132static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2133{
2134 /* Calculate the DMIF Request Bandwidth */
2135 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2136 fixed20_12 disp_clk, sclk, bandwidth;
2137 fixed20_12 a, b1, b2;
2138 u32 min_bandwidth;
2139
2140 a.full = dfixed_const(1000);
2141 disp_clk.full = dfixed_const(wm->disp_clk);
2142 disp_clk.full = dfixed_div(disp_clk, a);
2143 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2144 b1.full = dfixed_mul(a, disp_clk);
2145
2146 a.full = dfixed_const(1000);
2147 sclk.full = dfixed_const(wm->sclk);
2148 sclk.full = dfixed_div(sclk, a);
2149 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2150 b2.full = dfixed_mul(a, sclk);
2151
2152 a.full = dfixed_const(10);
2153 disp_clk_request_efficiency.full = dfixed_const(8);
2154 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2155
2156 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2157
2158 a.full = dfixed_const(min_bandwidth);
2159 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2160
2161 return dfixed_trunc(bandwidth);
2162}
2163
2164static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2165{
2166 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2167 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2168 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2169 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2170
2171 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2172}
2173
2174static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2175{
2176 /* Calculate the display mode Average Bandwidth
2177 * DisplayMode should contain the source and destination dimensions,
2178 * timing, etc.
2179 */
2180 fixed20_12 bpp;
2181 fixed20_12 line_time;
2182 fixed20_12 src_width;
2183 fixed20_12 bandwidth;
2184 fixed20_12 a;
2185
2186 a.full = dfixed_const(1000);
2187 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2188 line_time.full = dfixed_div(line_time, a);
2189 bpp.full = dfixed_const(wm->bytes_per_pixel);
2190 src_width.full = dfixed_const(wm->src_width);
2191 bandwidth.full = dfixed_mul(src_width, bpp);
2192 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2193 bandwidth.full = dfixed_div(bandwidth, line_time);
2194
2195 return dfixed_trunc(bandwidth);
2196}
2197
2198static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2199{
2200 /* First calcualte the latency in ns */
2201 u32 mc_latency = 2000; /* 2000 ns. */
2202 u32 available_bandwidth = dce6_available_bandwidth(wm);
2203 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2204 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2205 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2206 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2207 (wm->num_heads * cursor_line_pair_return_time);
2208 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2209 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2210 u32 tmp, dmif_size = 12288;
2211 fixed20_12 a, b, c;
2212
2213 if (wm->num_heads == 0)
2214 return 0;
2215
2216 a.full = dfixed_const(2);
2217 b.full = dfixed_const(1);
2218 if ((wm->vsc.full > a.full) ||
2219 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2220 (wm->vtaps >= 5) ||
2221 ((wm->vsc.full >= a.full) && wm->interlaced))
2222 max_src_lines_per_dst_line = 4;
2223 else
2224 max_src_lines_per_dst_line = 2;
2225
2226 a.full = dfixed_const(available_bandwidth);
2227 b.full = dfixed_const(wm->num_heads);
2228 a.full = dfixed_div(a, b);
ae45bbc2
MK		 2229 tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
2230 tmp = min(dfixed_trunc(a), tmp);
43b3cd99 2231
ae45bbc2 2232 lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
43b3cd99
AD		 2233
2234 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2235 b.full = dfixed_const(1000);
2236 c.full = dfixed_const(lb_fill_bw);
2237 b.full = dfixed_div(c, b);
2238 a.full = dfixed_div(a, b);
2239 line_fill_time = dfixed_trunc(a);
2240
2241 if (line_fill_time < wm->active_time)
2242 return latency;
2243 else
2244 return latency + (line_fill_time - wm->active_time);
2245
2246}
2247
2248static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2249{
2250 if (dce6_average_bandwidth(wm) <=
2251 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2252 return true;
2253 else
2254 return false;
2255};
2256
2257static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2258{
2259 if (dce6_average_bandwidth(wm) <=
2260 (dce6_available_bandwidth(wm) / wm->num_heads))
2261 return true;
2262 else
2263 return false;
2264};
2265
2266static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2267{
2268 u32 lb_partitions = wm->lb_size / wm->src_width;
2269 u32 line_time = wm->active_time + wm->blank_time;
2270 u32 latency_tolerant_lines;
2271 u32 latency_hiding;
2272 fixed20_12 a;
2273
2274 a.full = dfixed_const(1);
2275 if (wm->vsc.full > a.full)
2276 latency_tolerant_lines = 1;
2277 else {
2278 if (lb_partitions <= (wm->vtaps + 1))
2279 latency_tolerant_lines = 1;
2280 else
2281 latency_tolerant_lines = 2;
2282 }
2283
2284 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2285
2286 if (dce6_latency_watermark(wm) <= latency_hiding)
2287 return true;
2288 else
2289 return false;
2290}
2291
2292static void dce6_program_watermarks(struct radeon_device *rdev,
2293 struct radeon_crtc *radeon_crtc,
2294 u32 lb_size, u32 num_heads)
2295{
2296 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2297 struct dce6_wm_params wm_low, wm_high;
2298 u32 dram_channels;
e6b9a6c8 2299 u32 active_time;
43b3cd99
AD		 2300 u32 line_time = 0;
2301 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2302 u32 priority_a_mark = 0, priority_b_mark = 0;
2303 u32 priority_a_cnt = PRIORITY_OFF;
2304 u32 priority_b_cnt = PRIORITY_OFF;
2305 u32 tmp, arb_control3;
2306 fixed20_12 a, b, c;
2307
2308 if (radeon_crtc->base.enabled && num_heads && mode) {
55f61a04
MK		 2309 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2310 (u32)mode->clock);
2311 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2312 (u32)mode->clock);
2313 line_time = min(line_time, (u32)65535);
43b3cd99
AD		 2314 priority_a_cnt = 0;
2315 priority_b_cnt = 0;
2316
ca7db22b 2317 if (rdev->family == CHIP_ARUBA)
c696e53f 2318 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2319 else
c696e53f
AD		 2320 dram_channels = si_get_number_of_dram_channels(rdev);
2321
2322 /* watermark for high clocks */
2323 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2324 wm_high.yclk =
2325 radeon_dpm_get_mclk(rdev, false) * 10;
2326 wm_high.sclk =
2327 radeon_dpm_get_sclk(rdev, false) * 10;
2328 } else {
2329 wm_high.yclk = rdev->pm.current_mclk * 10;
2330 wm_high.sclk = rdev->pm.current_sclk * 10;
2331 }
2332
2333 wm_high.disp_clk = mode->clock;
2334 wm_high.src_width = mode->crtc_hdisplay;
e6b9a6c8 2335 wm_high.active_time = active_time;
c696e53f
AD		 2336 wm_high.blank_time = line_time - wm_high.active_time;
2337 wm_high.interlaced = false;
2338 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2339 wm_high.interlaced = true;
2340 wm_high.vsc = radeon_crtc->vsc;
2341 wm_high.vtaps = 1;
2342 if (radeon_crtc->rmx_type != RMX_OFF)
2343 wm_high.vtaps = 2;
2344 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2345 wm_high.lb_size = lb_size;
2346 wm_high.dram_channels = dram_channels;
2347 wm_high.num_heads = num_heads;
2348
2349 /* watermark for low clocks */
2350 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2351 wm_low.yclk =
2352 radeon_dpm_get_mclk(rdev, true) * 10;
2353 wm_low.sclk =
2354 radeon_dpm_get_sclk(rdev, true) * 10;
2355 } else {
2356 wm_low.yclk = rdev->pm.current_mclk * 10;
2357 wm_low.sclk = rdev->pm.current_sclk * 10;
2358 }
2359
2360 wm_low.disp_clk = mode->clock;
2361 wm_low.src_width = mode->crtc_hdisplay;
e6b9a6c8 2362 wm_low.active_time = active_time;
c696e53f
AD		 2363 wm_low.blank_time = line_time - wm_low.active_time;
2364 wm_low.interlaced = false;
2365 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2366 wm_low.interlaced = true;
2367 wm_low.vsc = radeon_crtc->vsc;
2368 wm_low.vtaps = 1;
2369 if (radeon_crtc->rmx_type != RMX_OFF)
2370 wm_low.vtaps = 2;
2371 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2372 wm_low.lb_size = lb_size;
2373 wm_low.dram_channels = dram_channels;
2374 wm_low.num_heads = num_heads;
43b3cd99
AD		 2375
2376 /* set for high clocks */
c696e53f 2377 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2378 /* set for low clocks */
c696e53f 2379 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2380
2381 /* possibly force display priority to high */
2382 /* should really do this at mode validation time... */
c696e53f
AD		 2383 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2384 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2385 !dce6_check_latency_hiding(&wm_high) ||
2386 (rdev->disp_priority == 2)) {
2387 DRM_DEBUG_KMS("force priority to high\n");
2388 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2389 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2390 }
2391 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2392 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2393 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2394 (rdev->disp_priority == 2)) {
2395 DRM_DEBUG_KMS("force priority to high\n");
2396 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2397 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2398 }
2399
2400 a.full = dfixed_const(1000);
2401 b.full = dfixed_const(mode->clock);
2402 b.full = dfixed_div(b, a);
2403 c.full = dfixed_const(latency_watermark_a);
2404 c.full = dfixed_mul(c, b);
2405 c.full = dfixed_mul(c, radeon_crtc->hsc);
2406 c.full = dfixed_div(c, a);
2407 a.full = dfixed_const(16);
2408 c.full = dfixed_div(c, a);
2409 priority_a_mark = dfixed_trunc(c);
2410 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2411
2412 a.full = dfixed_const(1000);
2413 b.full = dfixed_const(mode->clock);
2414 b.full = dfixed_div(b, a);
2415 c.full = dfixed_const(latency_watermark_b);
2416 c.full = dfixed_mul(c, b);
2417 c.full = dfixed_mul(c, radeon_crtc->hsc);
2418 c.full = dfixed_div(c, a);
2419 a.full = dfixed_const(16);
2420 c.full = dfixed_div(c, a);
2421 priority_b_mark = dfixed_trunc(c);
2422 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
5b5561b3
MK		 2423
2424 /* Save number of lines the linebuffer leads before the scanout */
2425 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
43b3cd99
AD		 2426 }
2427
2428 /* select wm A */
2429 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2430 tmp = arb_control3;
2431 tmp &= ~LATENCY_WATERMARK_MASK(3);
2432 tmp |= LATENCY_WATERMARK_MASK(1);
2433 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2434 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2435 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2436 LATENCY_HIGH_WATERMARK(line_time)));
2437 /* select wm B */
2438 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2439 tmp &= ~LATENCY_WATERMARK_MASK(3);
2440 tmp |= LATENCY_WATERMARK_MASK(2);
2441 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2442 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2443 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2444 LATENCY_HIGH_WATERMARK(line_time)));
2445 /* restore original selection */
2446 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2447
2448 /* write the priority marks */
2449 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2450 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2451
7178d2a6
AD		 2452 /* save values for DPM */
2453 radeon_crtc->line_time = line_time;
2454 radeon_crtc->wm_high = latency_watermark_a;
2455 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2456}
2457
2458void dce6_bandwidth_update(struct radeon_device *rdev)
2459{
2460 struct drm_display_mode *mode0 = NULL;
2461 struct drm_display_mode *mode1 = NULL;
2462 u32 num_heads = 0, lb_size;
2463 int i;
2464
8efe82ca
AD		 2465 if (!rdev->mode_info.mode_config_initialized)
2466 return;
2467
43b3cd99
AD		 2468 radeon_update_display_priority(rdev);
2469
2470 for (i = 0; i < rdev->num_crtc; i++) {
2471 if (rdev->mode_info.crtcs[i]->base.enabled)
2472 num_heads++;
2473 }
2474 for (i = 0; i < rdev->num_crtc; i += 2) {
2475 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2476 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2477 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2478 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2479 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2480 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2481 }
2482}
2483
0a96d72b
AD		 2484/*
2485 * Core functions
2486 */
0a96d72b
AD		 2487static void si_tiling_mode_table_init(struct radeon_device *rdev)
2488{
102534b0
JP		 2489 u32 *tile = rdev->config.si.tile_mode_array;
2490 const u32 num_tile_mode_states =
2491 ARRAY_SIZE(rdev->config.si.tile_mode_array);
2492 u32 reg_offset, split_equal_to_row_size;
0a96d72b
AD		 2493
2494 switch (rdev->config.si.mem_row_size_in_kb) {
2495 case 1:
2496 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2497 break;
2498 case 2:
2499 default:
2500 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2501 break;
2502 case 4:
2503 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2504 break;
2505 }
2506
102534b0
JP		 2507 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2508 tile[reg_offset] = 0;
2509
2510 switch(rdev->family) {
2511 case CHIP_TAHITI:
2512 case CHIP_PITCAIRN:
2513 /* non-AA compressed depth or any compressed stencil */
2514 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2515 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2516 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2517 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2518 NUM_BANKS(ADDR_SURF_16_BANK) |
2519 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2520 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2521 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2522 /* 2xAA/4xAA compressed depth only */
2523 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2525 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2526 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2527 NUM_BANKS(ADDR_SURF_16_BANK) |
2528 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2529 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2530 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2531 /* 8xAA compressed depth only */
2532 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2533 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2534 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2535 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2536 NUM_BANKS(ADDR_SURF_16_BANK) |
2537 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2538 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2539 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2540 /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2541 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2542 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2543 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2544 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2545 NUM_BANKS(ADDR_SURF_16_BANK) |
2546 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2547 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2548 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2549 /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2550 tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2551 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2552 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2553 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2554 NUM_BANKS(ADDR_SURF_16_BANK) |
2555 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2556 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2557 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2558 /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2559 tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2560 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2562 TILE_SPLIT(split_equal_to_row_size) |
2563 NUM_BANKS(ADDR_SURF_16_BANK) |
2564 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2565 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2566 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2567 /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2568 tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2569 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2571 TILE_SPLIT(split_equal_to_row_size) |
2572 NUM_BANKS(ADDR_SURF_16_BANK) |
2573 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2574 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2575 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2576 /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2577 tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2578 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2579 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2580 TILE_SPLIT(split_equal_to_row_size) |
2581 NUM_BANKS(ADDR_SURF_16_BANK) |
2582 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2583 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2584 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2585 /* 1D and 1D Array Surfaces */
2586 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2587 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2588 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2589 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2590 NUM_BANKS(ADDR_SURF_16_BANK) |
2591 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2593 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2594 /* Displayable maps. */
2595 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2596 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2598 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2599 NUM_BANKS(ADDR_SURF_16_BANK) |
2600 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2601 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2602 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2603 /* Display 8bpp. */
2604 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2607 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2608 NUM_BANKS(ADDR_SURF_16_BANK) |
2609 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2610 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2611 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2612 /* Display 16bpp. */
2613 tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2614 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2615 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2616 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2617 NUM_BANKS(ADDR_SURF_16_BANK) |
2618 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2620 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2621 /* Display 32bpp. */
2622 tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2623 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2624 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2625 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2626 NUM_BANKS(ADDR_SURF_16_BANK) |
2627 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2630 /* Thin. */
2631 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2632 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2633 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2634 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2635 NUM_BANKS(ADDR_SURF_16_BANK) |
2636 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2638 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2639 /* Thin 8 bpp. */
2640 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2643 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2644 NUM_BANKS(ADDR_SURF_16_BANK) |
2645 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2646 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2647 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2648 /* Thin 16 bpp. */
2649 tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2650 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2651 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2652 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2653 NUM_BANKS(ADDR_SURF_16_BANK) |
2654 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2657 /* Thin 32 bpp. */
2658 tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2659 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2660 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2661 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2662 NUM_BANKS(ADDR_SURF_16_BANK) |
2663 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2664 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2665 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2666 /* Thin 64 bpp. */
2667 tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2668 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2669 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2670 TILE_SPLIT(split_equal_to_row_size) |
2671 NUM_BANKS(ADDR_SURF_16_BANK) |
2672 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2674 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2675 /* 8 bpp PRT. */
2676 tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2677 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2678 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2679 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2680 NUM_BANKS(ADDR_SURF_16_BANK) |
2681 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2682 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2683 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2684 /* 16 bpp PRT */
2685 tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2686 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2687 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2688 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2689 NUM_BANKS(ADDR_SURF_16_BANK) |
2690 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2693 /* 32 bpp PRT */
2694 tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2695 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2696 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2697 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2698 NUM_BANKS(ADDR_SURF_16_BANK) |
2699 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2700 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2701 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2702 /* 64 bpp PRT */
2703 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2704 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2705 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2706 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2707 NUM_BANKS(ADDR_SURF_16_BANK) |
2708 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2711 /* 128 bpp PRT */
2712 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2713 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2714 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2715 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2716 NUM_BANKS(ADDR_SURF_8_BANK) |
2717 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2719 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2720
2721 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2722 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2723 break;
2724
2725 case CHIP_VERDE:
2726 case CHIP_OLAND:
2727 case CHIP_HAINAN:
2728 /* non-AA compressed depth or any compressed stencil */
2729 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2730 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2731 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2732 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2733 NUM_BANKS(ADDR_SURF_16_BANK) |
2734 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2735 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2736 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2737 /* 2xAA/4xAA compressed depth only */
2738 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2739 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2740 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2741 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2742 NUM_BANKS(ADDR_SURF_16_BANK) |
2743 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2746 /* 8xAA compressed depth only */
2747 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2748 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2749 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2750 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2751 NUM_BANKS(ADDR_SURF_16_BANK) |
2752 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2753 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2754 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2755 /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2756 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2757 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2758 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2760 NUM_BANKS(ADDR_SURF_16_BANK) |
2761 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2764 /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2765 tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2766 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2767 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2768 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2769 NUM_BANKS(ADDR_SURF_16_BANK) |
2770 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2771 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2772 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2773 /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2774 tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2775 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2776 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2777 TILE_SPLIT(split_equal_to_row_size) |
2778 NUM_BANKS(ADDR_SURF_16_BANK) |
2779 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2780 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2781 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2782 /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2783 tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2784 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2785 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2786 TILE_SPLIT(split_equal_to_row_size) |
2787 NUM_BANKS(ADDR_SURF_16_BANK) |
2788 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2789 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2790 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2791 /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2792 tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2793 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2794 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2795 TILE_SPLIT(split_equal_to_row_size) |
2796 NUM_BANKS(ADDR_SURF_16_BANK) |
2797 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2798 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2799 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2800 /* 1D and 1D Array Surfaces */
2801 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2802 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2803 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2804 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2805 NUM_BANKS(ADDR_SURF_16_BANK) |
2806 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2807 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2808 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2809 /* Displayable maps. */
2810 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2811 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2812 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2813 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2814 NUM_BANKS(ADDR_SURF_16_BANK) |
2815 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2816 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2817 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2818 /* Display 8bpp. */
2819 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2822 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2823 NUM_BANKS(ADDR_SURF_16_BANK) |
2824 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2825 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2826 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2827 /* Display 16bpp. */
2828 tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2829 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2830 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2831 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2832 NUM_BANKS(ADDR_SURF_16_BANK) |
2833 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2834 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2835 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2836 /* Display 32bpp. */
2837 tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2840 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2841 NUM_BANKS(ADDR_SURF_16_BANK) |
2842 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2843 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2844 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2845 /* Thin. */
2846 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2847 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2848 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2849 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2850 NUM_BANKS(ADDR_SURF_16_BANK) |
2851 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2852 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2853 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2854 /* Thin 8 bpp. */
2855 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2856 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2857 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2858 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2859 NUM_BANKS(ADDR_SURF_16_BANK) |
2860 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2861 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2862 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2863 /* Thin 16 bpp. */
2864 tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2865 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2866 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2867 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2868 NUM_BANKS(ADDR_SURF_16_BANK) |
2869 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2870 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2871 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2872 /* Thin 32 bpp. */
2873 tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2874 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2875 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2876 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2877 NUM_BANKS(ADDR_SURF_16_BANK) |
2878 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2879 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2880 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2881 /* Thin 64 bpp. */
2882 tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2883 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2884 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2885 TILE_SPLIT(split_equal_to_row_size) |
2886 NUM_BANKS(ADDR_SURF_16_BANK) |
2887 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2888 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2889 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2890 /* 8 bpp PRT. */
2891 tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2895 NUM_BANKS(ADDR_SURF_16_BANK) |
2896 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2897 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2898 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2899 /* 16 bpp PRT */
2900 tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2901 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2902 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2903 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2904 NUM_BANKS(ADDR_SURF_16_BANK) |
2905 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2906 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2907 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2908 /* 32 bpp PRT */
2909 tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2910 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2911 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2912 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2913 NUM_BANKS(ADDR_SURF_16_BANK) |
2914 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2917 /* 64 bpp PRT */
2918 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2919 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2920 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2921 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2922 NUM_BANKS(ADDR_SURF_16_BANK) |
2923 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2924 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2925 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2926 /* 128 bpp PRT */
2927 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2928 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2929 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2930 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2931 NUM_BANKS(ADDR_SURF_8_BANK) |
2932 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2933 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2934 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2935
2936 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2937 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2938 break;
2939
2940 default:
0a96d72b 2941 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
102534b0 2942 }
0a96d72b
AD		 2943}
2944
1a8ca750
AD		 2945static void si_select_se_sh(struct radeon_device *rdev,
2946 u32 se_num, u32 sh_num)
2947{
2948 u32 data = INSTANCE_BROADCAST_WRITES;
2949
2950 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2951 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2952 else if (se_num == 0xffffffff)
2953 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2954 else if (sh_num == 0xffffffff)
2955 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2956 else
2957 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2958 WREG32(GRBM_GFX_INDEX, data);
2959}
2960
2961static u32 si_create_bitmask(u32 bit_width)
2962{
2963 u32 i, mask = 0;
2964
2965 for (i = 0; i < bit_width; i++) {
2966 mask <<= 1;
2967 mask |= 1;
2968 }
2969 return mask;
2970}
2971
2972static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2973{
2974 u32 data, mask;
2975
2976 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2977 if (data & 1)
2978 data &= INACTIVE_CUS_MASK;
2979 else
2980 data = 0;
2981 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2982
2983 data >>= INACTIVE_CUS_SHIFT;
2984
2985 mask = si_create_bitmask(cu_per_sh);
2986
2987 return ~data & mask;
2988}
2989
2990static void si_setup_spi(struct radeon_device *rdev,
2991 u32 se_num, u32 sh_per_se,
2992 u32 cu_per_sh)
2993{
2994 int i, j, k;
2995 u32 data, mask, active_cu;
2996
2997 for (i = 0; i < se_num; i++) {
2998 for (j = 0; j < sh_per_se; j++) {
2999 si_select_se_sh(rdev, i, j);
3000 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
3001 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
3002
3003 mask = 1;
3004 for (k = 0; k < 16; k++) {
3005 mask <<= k;
3006 if (active_cu & mask) {
3007 data &= ~mask;
3008 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
3009 break;
3010 }
3011 }
3012 }
3013 }
3014 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3015}
3016
3017static u32 si_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3018 u32 max_rb_num_per_se,
1a8ca750
AD		 3019 u32 sh_per_se)
3020{
3021 u32 data, mask;
3022
3023 data = RREG32(CC_RB_BACKEND_DISABLE);
3024 if (data & 1)
3025 data &= BACKEND_DISABLE_MASK;
3026 else
3027 data = 0;
3028 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3029
3030 data >>= BACKEND_DISABLE_SHIFT;
3031
9fadb352 3032 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
1a8ca750
AD		 3033
3034 return data & mask;
3035}
3036
3037static void si_setup_rb(struct radeon_device *rdev,
3038 u32 se_num, u32 sh_per_se,
9fadb352 3039 u32 max_rb_num_per_se)
1a8ca750
AD		 3040{
3041 int i, j;
3042 u32 data, mask;
3043 u32 disabled_rbs = 0;
3044 u32 enabled_rbs = 0;
3045
3046 for (i = 0; i < se_num; i++) {
3047 for (j = 0; j < sh_per_se; j++) {
3048 si_select_se_sh(rdev, i, j);
9fadb352 3049 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
1a8ca750
AD		 3050 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3051 }
3052 }
3053 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3054
3055 mask = 1;
9fadb352 3056 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
1a8ca750
AD		 3057 if (!(disabled_rbs & mask))
3058 enabled_rbs |= mask;
3059 mask <<= 1;
3060 }
3061
439a1cff
MO		 3062 rdev->config.si.backend_enable_mask = enabled_rbs;
3063
1a8ca750
AD		 3064 for (i = 0; i < se_num; i++) {
3065 si_select_se_sh(rdev, i, 0xffffffff);
3066 data = 0;
3067 for (j = 0; j < sh_per_se; j++) {
3068 switch (enabled_rbs & 3) {
3069 case 1:
3070 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3071 break;
3072 case 2:
3073 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3074 break;
3075 case 3:
3076 default:
3077 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3078 break;
3079 }
3080 enabled_rbs >>= 2;
3081 }
3082 WREG32(PA_SC_RASTER_CONFIG, data);
3083 }
3084 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3085}
3086
0a96d72b
AD		 3087static void si_gpu_init(struct radeon_device *rdev)
3088{
0a96d72b 3089 u32 gb_addr_config = 0;
c317fcbb 3090 u32 mc_arb_ramcfg;
0a96d72b 3091 u32 sx_debug_1;
0a96d72b
AD		 3092 u32 hdp_host_path_cntl;
3093 u32 tmp;
6101b3ae 3094 int i, j;
0a96d72b
AD		 3095
3096 switch (rdev->family) {
3097 case CHIP_TAHITI:
3098 rdev->config.si.max_shader_engines = 2;
0a96d72b 3099 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 3100 rdev->config.si.max_cu_per_sh = 8;
3101 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 3102 rdev->config.si.max_backends_per_se = 4;
3103 rdev->config.si.max_texture_channel_caches = 12;
3104 rdev->config.si.max_gprs = 256;
3105 rdev->config.si.max_gs_threads = 32;
3106 rdev->config.si.max_hw_contexts = 8;
3107
3108 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3109 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3110 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3111 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 3112 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 3113 break;
3114 case CHIP_PITCAIRN:
3115 rdev->config.si.max_shader_engines = 2;
0a96d72b 3116 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 3117 rdev->config.si.max_cu_per_sh = 5;
3118 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 3119 rdev->config.si.max_backends_per_se = 4;
3120 rdev->config.si.max_texture_channel_caches = 8;
3121 rdev->config.si.max_gprs = 256;
3122 rdev->config.si.max_gs_threads = 32;
3123 rdev->config.si.max_hw_contexts = 8;
3124
3125 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3126 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3127 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3128 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 3129 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 3130 break;
3131 case CHIP_VERDE:
3132 default:
3133 rdev->config.si.max_shader_engines = 1;
0a96d72b 3134 rdev->config.si.max_tile_pipes = 4;
468ef1a5 3135 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 3136 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 3137 rdev->config.si.max_backends_per_se = 4;
3138 rdev->config.si.max_texture_channel_caches = 4;
3139 rdev->config.si.max_gprs = 256;
3140 rdev->config.si.max_gs_threads = 32;
3141 rdev->config.si.max_hw_contexts = 8;
3142
d0ae7fcc
AD		 3143 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3144 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3145 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3146 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3147 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3148 break;
3149 case CHIP_OLAND:
3150 rdev->config.si.max_shader_engines = 1;
3151 rdev->config.si.max_tile_pipes = 4;
3152 rdev->config.si.max_cu_per_sh = 6;
3153 rdev->config.si.max_sh_per_se = 1;
3154 rdev->config.si.max_backends_per_se = 2;
3155 rdev->config.si.max_texture_channel_caches = 4;
3156 rdev->config.si.max_gprs = 256;
3157 rdev->config.si.max_gs_threads = 16;
3158 rdev->config.si.max_hw_contexts = 8;
3159
0a96d72b
AD		 3160 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3161 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3162 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3163 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 3164 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 3165 break;
8b02859d
AD		 3166 case CHIP_HAINAN:
3167 rdev->config.si.max_shader_engines = 1;
3168 rdev->config.si.max_tile_pipes = 4;
3169 rdev->config.si.max_cu_per_sh = 5;
3170 rdev->config.si.max_sh_per_se = 1;
3171 rdev->config.si.max_backends_per_se = 1;
3172 rdev->config.si.max_texture_channel_caches = 2;
3173 rdev->config.si.max_gprs = 256;
3174 rdev->config.si.max_gs_threads = 16;
3175 rdev->config.si.max_hw_contexts = 8;
3176
3177 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3178 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3179 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3180 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3181 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3182 break;
0a96d72b
AD		 3183 }
3184
3185 /* Initialize HDP */
3186 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3187 WREG32((0x2c14 + j), 0x00000000);
3188 WREG32((0x2c18 + j), 0x00000000);
3189 WREG32((0x2c1c + j), 0x00000000);
3190 WREG32((0x2c20 + j), 0x00000000);
3191 WREG32((0x2c24 + j), 0x00000000);
3192 }
3193
3194 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
18ad01ef
CK		 3195 WREG32(SRBM_INT_CNTL, 1);
3196 WREG32(SRBM_INT_ACK, 1);
0a96d72b
AD		 3197
3198 evergreen_fix_pci_max_read_req_size(rdev);
3199
3200 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3201
c317fcbb 3202 RREG32(MC_SHARED_CHMAP);
0a96d72b
AD		 3203 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3204
0a96d72b 3205 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 3206 rdev->config.si.mem_max_burst_length_bytes = 256;
3207 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3208 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3209 if (rdev->config.si.mem_row_size_in_kb > 4)
3210 rdev->config.si.mem_row_size_in_kb = 4;
3211 /* XXX use MC settings? */
3212 rdev->config.si.shader_engine_tile_size = 32;
3213 rdev->config.si.num_gpus = 1;
3214 rdev->config.si.multi_gpu_tile_size = 64;
3215
1a8ca750
AD		 3216 /* fix up row size */
3217 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 3218 switch (rdev->config.si.mem_row_size_in_kb) {
3219 case 1:
3220 default:
3221 gb_addr_config |= ROW_SIZE(0);
3222 break;
3223 case 2:
3224 gb_addr_config |= ROW_SIZE(1);
3225 break;
3226 case 4:
3227 gb_addr_config |= ROW_SIZE(2);
3228 break;
3229 }
3230
0a96d72b
AD		 3231 /* setup tiling info dword. gb_addr_config is not adequate since it does
3232 * not have bank info, so create a custom tiling dword.
3233 * bits 3:0 num_pipes
3234 * bits 7:4 num_banks
3235 * bits 11:8 group_size
3236 * bits 15:12 row_size
3237 */
3238 rdev->config.si.tile_config = 0;
3239 switch (rdev->config.si.num_tile_pipes) {
3240 case 1:
3241 rdev->config.si.tile_config |= (0 << 0);
3242 break;
3243 case 2:
3244 rdev->config.si.tile_config |= (1 << 0);
3245 break;
3246 case 4:
3247 rdev->config.si.tile_config |= (2 << 0);
3248 break;
3249 case 8:
3250 default:
3251 /* XXX what about 12? */
3252 rdev->config.si.tile_config |= (3 << 0);
3253 break;
3d581b11 3254 }
dca571a6
CK		 3255 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3256 case 0: /* four banks */
1a8ca750 3257 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3258 break;
3259 case 1: /* eight banks */
3260 rdev->config.si.tile_config |= 1 << 4;
3261 break;
3262 case 2: /* sixteen banks */
3263 default:
3264 rdev->config.si.tile_config |= 2 << 4;
3265 break;
3266 }
0a96d72b
AD		 3267 rdev->config.si.tile_config |=
3268 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3269 rdev->config.si.tile_config |=
3270 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3271
0a96d72b
AD		 3272 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3273 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3274 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3275 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3276 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3277 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3278 if (rdev->has_uvd) {
3279 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3280 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3281 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3282 }
0a96d72b 3283
1a8ca750 3284 si_tiling_mode_table_init(rdev);
0a96d72b 3285
1a8ca750
AD		 3286 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3287 rdev->config.si.max_sh_per_se,
3288 rdev->config.si.max_backends_per_se);
0a96d72b 3289
1a8ca750
AD		 3290 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3291 rdev->config.si.max_sh_per_se,
3292 rdev->config.si.max_cu_per_sh);
0a96d72b 3293
52da51f0 3294 rdev->config.si.active_cus = 0;
65fcf668
AD		 3295 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3296 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
6101b3ae
AD		 3297 rdev->config.si.active_cus +=
3298 hweight32(si_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3299 }
3300 }
0a96d72b
AD		 3301
3302 /* set HW defaults for 3D engine */
3303 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3304 ROQ_IB2_START(0x2b)));
3305 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3306
3307 sx_debug_1 = RREG32(SX_DEBUG_1);
3308 WREG32(SX_DEBUG_1, sx_debug_1);
3309
3310 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3311
3312 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3313 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3314 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3315 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3316
3317 WREG32(VGT_NUM_INSTANCES, 1);
3318
3319 WREG32(CP_PERFMON_CNTL, 0);
3320
3321 WREG32(SQ_CONFIG, 0);
3322
3323 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3324 FORCE_EOV_MAX_REZ_CNT(255)));
3325
3326 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3327 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3328
3329 WREG32(VGT_GS_VERTEX_REUSE, 16);
3330 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3331
3332 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3333 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3334 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3335 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3336 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3337 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3338 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3339 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3340
3341 tmp = RREG32(HDP_MISC_CNTL);
3342 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3343 WREG32(HDP_MISC_CNTL, tmp);
3344
3345 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3346 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3347
3348 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3349
3350 udelay(50);
3351}
c476dde2 3352
2ece2e8b
AD		 3353/*
3354 * GPU scratch registers helpers function.
3355 */
3356static void si_scratch_init(struct radeon_device *rdev)
3357{
3358 int i;
3359
3360 rdev->scratch.num_reg = 7;
3361 rdev->scratch.reg_base = SCRATCH_REG0;
3362 for (i = 0; i < rdev->scratch.num_reg; i++) {
3363 rdev->scratch.free[i] = true;
3364 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3365 }
3366}
3367
3368void si_fence_ring_emit(struct radeon_device *rdev,
3369 struct radeon_fence *fence)
3370{
3371 struct radeon_ring *ring = &rdev->ring[fence->ring];
3372 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3373
3374 /* flush read cache over gart */
3375 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3376 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3377 radeon_ring_write(ring, 0);
3378 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3379 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3380 PACKET3_TC_ACTION_ENA |
3381 PACKET3_SH_KCACHE_ACTION_ENA |
3382 PACKET3_SH_ICACHE_ACTION_ENA);
3383 radeon_ring_write(ring, 0xFFFFFFFF);
3384 radeon_ring_write(ring, 0);
3385 radeon_ring_write(ring, 10); /* poll interval */
3386 /* EVENT_WRITE_EOP - flush caches, send int */
3387 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3388 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
5e167cdb 3389 radeon_ring_write(ring, lower_32_bits(addr));
2ece2e8b
AD		 3390 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3391 radeon_ring_write(ring, fence->seq);
3392 radeon_ring_write(ring, 0);
3393}
3394
3395/*
3396 * IB stuff
3397 */
3398void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3399{
876dc9f3 3400 struct radeon_ring *ring = &rdev->ring[ib->ring];
7c42bc1a 3401 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
2ece2e8b
AD		 3402 u32 header;
3403
a85a7da4
AD		 3404 if (ib->is_const_ib) {
3405 /* set switch buffer packet before const IB */
3406 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3407 radeon_ring_write(ring, 0);
45df6803 3408
2ece2e8b 3409 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3410 } else {
89d35807 3411 u32 next_rptr;
a85a7da4 3412 if (ring->rptr_save_reg) {
89d35807 3413 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3414 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3415 radeon_ring_write(ring, ((ring->rptr_save_reg -
3416 PACKET3_SET_CONFIG_REG_START) >> 2));
3417 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3418 } else if (rdev->wb.enabled) {
3419 next_rptr = ring->wptr + 5 + 4 + 8;
3420 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3421 radeon_ring_write(ring, (1 << 8));
3422 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 3423 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
89d35807 3424 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3425 }
3426
2ece2e8b 3427 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3428 }
2ece2e8b
AD		 3429
3430 radeon_ring_write(ring, header);
3431 radeon_ring_write(ring,
3432#ifdef __BIG_ENDIAN
3433 (2 << 0) |
3434#endif
3435 (ib->gpu_addr & 0xFFFFFFFC));
3436 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
7c42bc1a 3437 radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
2ece2e8b 3438
a85a7da4
AD		 3439 if (!ib->is_const_ib) {
3440 /* flush read cache over gart for this vmid */
3441 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3442 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
7c42bc1a 3443 radeon_ring_write(ring, vm_id);
a85a7da4
AD		 3444 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3445 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3446 PACKET3_TC_ACTION_ENA |
3447 PACKET3_SH_KCACHE_ACTION_ENA |
3448 PACKET3_SH_ICACHE_ACTION_ENA);
3449 radeon_ring_write(ring, 0xFFFFFFFF);
3450 radeon_ring_write(ring, 0);
3451 radeon_ring_write(ring, 10); /* poll interval */
3452 }
2ece2e8b
AD		 3453}
3454
48c0c902
AD		 3455/*
3456 * CP.
3457 */
3458static void si_cp_enable(struct radeon_device *rdev, bool enable)
3459{
3460 if (enable)
3461 WREG32(CP_ME_CNTL, 0);
3462 else {
50efa51a
AD		 3463 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3464 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
48c0c902
AD		 3465 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3466 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3467 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3468 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3469 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3470 }
3471 udelay(50);
3472}
3473
3474static int si_cp_load_microcode(struct radeon_device *rdev)
3475{
48c0c902
AD		 3476 int i;
3477
629bd33c 3478 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
48c0c902
AD		 3479 return -EINVAL;
3480
3481 si_cp_enable(rdev, false);
3482
629bd33c
AD		 3483 if (rdev->new_fw) {
3484 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3485 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3486 const struct gfx_firmware_header_v1_0 *ce_hdr =
3487 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3488 const struct gfx_firmware_header_v1_0 *me_hdr =
3489 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3490 const __le32 *fw_data;
3491 u32 fw_size;
3492
3493 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3494 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3495 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3496
3497 /* PFP */
3498 fw_data = (const __le32 *)
3499 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3500 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3501 WREG32(CP_PFP_UCODE_ADDR, 0);
3502 for (i = 0; i < fw_size; i++)
3503 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3504 WREG32(CP_PFP_UCODE_ADDR, 0);
3505
3506 /* CE */
3507 fw_data = (const __le32 *)
3508 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3509 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3510 WREG32(CP_CE_UCODE_ADDR, 0);
3511 for (i = 0; i < fw_size; i++)
3512 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3513 WREG32(CP_CE_UCODE_ADDR, 0);
3514
3515 /* ME */
3516 fw_data = (const __be32 *)
3517 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3518 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3519 WREG32(CP_ME_RAM_WADDR, 0);
3520 for (i = 0; i < fw_size; i++)
3521 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3522 WREG32(CP_ME_RAM_WADDR, 0);
3523 } else {
3524 const __be32 *fw_data;
3525
3526 /* PFP */
3527 fw_data = (const __be32 *)rdev->pfp_fw->data;
3528 WREG32(CP_PFP_UCODE_ADDR, 0);
3529 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3530 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3531 WREG32(CP_PFP_UCODE_ADDR, 0);
3532
3533 /* CE */
3534 fw_data = (const __be32 *)rdev->ce_fw->data;
3535 WREG32(CP_CE_UCODE_ADDR, 0);
3536 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3537 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3538 WREG32(CP_CE_UCODE_ADDR, 0);
3539
3540 /* ME */
3541 fw_data = (const __be32 *)rdev->me_fw->data;
3542 WREG32(CP_ME_RAM_WADDR, 0);
3543 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3544 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3545 WREG32(CP_ME_RAM_WADDR, 0);
3546 }
48c0c902
AD		 3547
3548 WREG32(CP_PFP_UCODE_ADDR, 0);
3549 WREG32(CP_CE_UCODE_ADDR, 0);
3550 WREG32(CP_ME_RAM_WADDR, 0);
3551 WREG32(CP_ME_RAM_RADDR, 0);
3552 return 0;
3553}
3554
3555static int si_cp_start(struct radeon_device *rdev)
3556{
3557 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3558 int r, i;
3559
3560 r = radeon_ring_lock(rdev, ring, 7 + 4);
3561 if (r) {
3562 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3563 return r;
3564 }
3565 /* init the CP */
3566 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3567 radeon_ring_write(ring, 0x1);
3568 radeon_ring_write(ring, 0x0);
3569 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3570 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3571 radeon_ring_write(ring, 0);
3572 radeon_ring_write(ring, 0);
3573
3574 /* init the CE partitions */
3575 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3576 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3577 radeon_ring_write(ring, 0xc000);
3578 radeon_ring_write(ring, 0xe000);
1538a9e0 3579 radeon_ring_unlock_commit(rdev, ring, false);
48c0c902
AD		 3580
3581 si_cp_enable(rdev, true);
3582
3583 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3584 if (r) {
3585 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3586 return r;
3587 }
3588
3589 /* setup clear context state */
3590 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3591 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3592
3593 for (i = 0; i < si_default_size; i++)
3594 radeon_ring_write(ring, si_default_state[i]);
3595
3596 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3597 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3598
3599 /* set clear context state */
3600 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3601 radeon_ring_write(ring, 0);
3602
3603 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3604 radeon_ring_write(ring, 0x00000316);
3605 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3606 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3607
1538a9e0 3608 radeon_ring_unlock_commit(rdev, ring, false);
48c0c902
AD		 3609
3610 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3611 ring = &rdev->ring[i];
3612 r = radeon_ring_lock(rdev, ring, 2);
3613
3614 /* clear the compute context state */
3615 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3616 radeon_ring_write(ring, 0);
3617
1538a9e0 3618 radeon_ring_unlock_commit(rdev, ring, false);
48c0c902
AD		 3619 }
3620
3621 return 0;
3622}
3623
3624static void si_cp_fini(struct radeon_device *rdev)
3625{
45df6803 3626 struct radeon_ring *ring;
48c0c902 3627 si_cp_enable(rdev, false);
45df6803
CK		 3628
3629 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3630 radeon_ring_fini(rdev, ring);
3631 radeon_scratch_free(rdev, ring->rptr_save_reg);
3632
3633 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3634 radeon_ring_fini(rdev, ring);
3635 radeon_scratch_free(rdev, ring->rptr_save_reg);
3636
3637 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3638 radeon_ring_fini(rdev, ring);
3639 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3640}
3641
3642static int si_cp_resume(struct radeon_device *rdev)
3643{
3644 struct radeon_ring *ring;
3645 u32 tmp;
3646 u32 rb_bufsz;
3647 int r;
3648
811e4d58
AD		 3649 si_enable_gui_idle_interrupt(rdev, false);
3650
48c0c902
AD		 3651 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3652 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3653
3654 /* Set the write pointer delay */
3655 WREG32(CP_RB_WPTR_DELAY, 0);
3656
3657 WREG32(CP_DEBUG, 0);
3658 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3659
3660 /* ring 0 - compute and gfx */
3661 /* Set ring buffer size */
3662 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3663 rb_bufsz = order_base_2(ring->ring_size / 8);
3664 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3665#ifdef __BIG_ENDIAN
3666 tmp |= BUF_SWAP_32BIT;
3667#endif
3668 WREG32(CP_RB0_CNTL, tmp);
3669
3670 /* Initialize the ring buffer's read and write pointers */
3671 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3672 ring->wptr = 0;
3673 WREG32(CP_RB0_WPTR, ring->wptr);
3674
48fc7f7e 3675 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3676 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3677 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3678
3679 if (rdev->wb.enabled)
3680 WREG32(SCRATCH_UMSK, 0xff);
3681 else {
3682 tmp |= RB_NO_UPDATE;
3683 WREG32(SCRATCH_UMSK, 0);
3684 }
3685
3686 mdelay(1);
3687 WREG32(CP_RB0_CNTL, tmp);
3688
3689 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3690
48c0c902
AD		 3691 /* ring1 - compute only */
3692 /* Set ring buffer size */
3693 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
b72a8925
DV		 3694 rb_bufsz = order_base_2(ring->ring_size / 8);
3695 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3696#ifdef __BIG_ENDIAN
3697 tmp |= BUF_SWAP_32BIT;
3698#endif
3699 WREG32(CP_RB1_CNTL, tmp);
3700
3701 /* Initialize the ring buffer's read and write pointers */
3702 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3703 ring->wptr = 0;
3704 WREG32(CP_RB1_WPTR, ring->wptr);
3705
48fc7f7e 3706 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3707 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3708 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3709
3710 mdelay(1);
3711 WREG32(CP_RB1_CNTL, tmp);
3712
3713 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3714
48c0c902
AD		 3715 /* ring2 - compute only */
3716 /* Set ring buffer size */
3717 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
b72a8925
DV		 3718 rb_bufsz = order_base_2(ring->ring_size / 8);
3719 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3720#ifdef __BIG_ENDIAN
3721 tmp |= BUF_SWAP_32BIT;
3722#endif
3723 WREG32(CP_RB2_CNTL, tmp);
3724
3725 /* Initialize the ring buffer's read and write pointers */
3726 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3727 ring->wptr = 0;
3728 WREG32(CP_RB2_WPTR, ring->wptr);
3729
48fc7f7e 3730 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3731 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3732 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3733
3734 mdelay(1);
3735 WREG32(CP_RB2_CNTL, tmp);
3736
3737 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3738
48c0c902
AD		 3739 /* start the rings */
3740 si_cp_start(rdev);
3741 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3742 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3743 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3744 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3745 if (r) {
3746 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3747 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3748 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3749 return r;
3750 }
3751 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3752 if (r) {
3753 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3754 }
3755 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3756 if (r) {
3757 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3758 }
3759
811e4d58
AD		 3760 si_enable_gui_idle_interrupt(rdev, true);
3761
50efa51a
AD		 3762 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3763 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3764
48c0c902
AD		 3765 return 0;
3766}
3767
2483b4ea 3768u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3769{
014bb209 3770 u32 reset_mask = 0;
1c534671 3771 u32 tmp;
06bc6df0 3772
014bb209
AD		 3773 /* GRBM_STATUS */
3774 tmp = RREG32(GRBM_STATUS);
3775 if (tmp & (PA_BUSY | SC_BUSY |
3776 BCI_BUSY | SX_BUSY |
3777 TA_BUSY | VGT_BUSY |
3778 DB_BUSY | CB_BUSY |
3779 GDS_BUSY | SPI_BUSY |
3780 IA_BUSY | IA_BUSY_NO_DMA))
3781 reset_mask |= RADEON_RESET_GFX;
3782
3783 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3784 CP_BUSY | CP_COHERENCY_BUSY))
3785 reset_mask |= RADEON_RESET_CP;
3786
3787 if (tmp & GRBM_EE_BUSY)
3788 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3789
3790 /* GRBM_STATUS2 */
3791 tmp = RREG32(GRBM_STATUS2);
3792 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3793 reset_mask |= RADEON_RESET_RLC;
3794
3795 /* DMA_STATUS_REG 0 */
3796 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3797 if (!(tmp & DMA_IDLE))
3798 reset_mask |= RADEON_RESET_DMA;
3799
3800 /* DMA_STATUS_REG 1 */
3801 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3802 if (!(tmp & DMA_IDLE))
3803 reset_mask |= RADEON_RESET_DMA1;
3804
3805 /* SRBM_STATUS2 */
3806 tmp = RREG32(SRBM_STATUS2);
3807 if (tmp & DMA_BUSY)
3808 reset_mask |= RADEON_RESET_DMA;
3809
3810 if (tmp & DMA1_BUSY)
3811 reset_mask |= RADEON_RESET_DMA1;
3812
3813 /* SRBM_STATUS */
3814 tmp = RREG32(SRBM_STATUS);
3815
3816 if (tmp & IH_BUSY)
3817 reset_mask |= RADEON_RESET_IH;
3818
3819 if (tmp & SEM_BUSY)
3820 reset_mask |= RADEON_RESET_SEM;
3821
3822 if (tmp & GRBM_RQ_PENDING)
3823 reset_mask |= RADEON_RESET_GRBM;
3824
3825 if (tmp & VMC_BUSY)
3826 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3827
014bb209
AD		 3828 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3829 MCC_BUSY | MCD_BUSY))
3830 reset_mask |= RADEON_RESET_MC;
3831
3832 if (evergreen_is_display_hung(rdev))
3833 reset_mask |= RADEON_RESET_DISPLAY;
3834
3835 /* VM_L2_STATUS */
3836 tmp = RREG32(VM_L2_STATUS);
3837 if (tmp & L2_BUSY)
3838 reset_mask |= RADEON_RESET_VMC;
3839
d808fc88
AD		 3840 /* Skip MC reset as it's mostly likely not hung, just busy */
3841 if (reset_mask & RADEON_RESET_MC) {
3842 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3843 reset_mask &= ~RADEON_RESET_MC;
3844 }
3845
014bb209
AD		 3846 return reset_mask;
3847}
3848
3849static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3850{
3851 struct evergreen_mc_save save;
3852 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3853 u32 tmp;
19fc42ed 3854
06bc6df0 3855 if (reset_mask == 0)
014bb209 3856 return;
06bc6df0
AD		 3857
3858 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3859
1c534671 3860 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3861 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3862 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3863 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3864 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3865
a6f4ae8d
AD		 3866 /* disable PG/CG */
3867 si_fini_pg(rdev);
3868 si_fini_cg(rdev);
3869
3870 /* stop the rlc */
3871 si_rlc_stop(rdev);
3872
1c534671
AD		 3873 /* Disable CP parsing/prefetching */
3874 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3875
3876 if (reset_mask & RADEON_RESET_DMA) {
3877 /* dma0 */
3878 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3879 tmp &= ~DMA_RB_ENABLE;
3880 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3881 }
3882 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3883 /* dma1 */
3884 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3885 tmp &= ~DMA_RB_ENABLE;
3886 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3887 }
3888
f770d78a
AD		 3889 udelay(50);
3890
3891 evergreen_mc_stop(rdev, &save);
3892 if (evergreen_mc_wait_for_idle(rdev)) {
3893 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3894 }
3895
1c534671
AD		 3896 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3897 grbm_soft_reset = SOFT_RESET_CB |
3898 SOFT_RESET_DB |
3899 SOFT_RESET_GDS |
3900 SOFT_RESET_PA |
3901 SOFT_RESET_SC |
3902 SOFT_RESET_BCI |
3903 SOFT_RESET_SPI |
3904 SOFT_RESET_SX |
3905 SOFT_RESET_TC |
3906 SOFT_RESET_TA |
3907 SOFT_RESET_VGT |
3908 SOFT_RESET_IA;
3909 }
3910
3911 if (reset_mask & RADEON_RESET_CP) {
3912 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3913
3914 srbm_soft_reset |= SOFT_RESET_GRBM;
3915 }
06bc6df0
AD		 3916
3917 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3918 srbm_soft_reset |= SOFT_RESET_DMA;
3919
3920 if (reset_mask & RADEON_RESET_DMA1)
3921 srbm_soft_reset |= SOFT_RESET_DMA1;
3922
3923 if (reset_mask & RADEON_RESET_DISPLAY)
3924 srbm_soft_reset |= SOFT_RESET_DC;
3925
3926 if (reset_mask & RADEON_RESET_RLC)
3927 grbm_soft_reset |= SOFT_RESET_RLC;
3928
3929 if (reset_mask & RADEON_RESET_SEM)
3930 srbm_soft_reset |= SOFT_RESET_SEM;
3931
3932 if (reset_mask & RADEON_RESET_IH)
3933 srbm_soft_reset |= SOFT_RESET_IH;
3934
3935 if (reset_mask & RADEON_RESET_GRBM)
3936 srbm_soft_reset |= SOFT_RESET_GRBM;
3937
3938 if (reset_mask & RADEON_RESET_VMC)
3939 srbm_soft_reset |= SOFT_RESET_VMC;
3940
3941 if (reset_mask & RADEON_RESET_MC)
3942 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3943
3944 if (grbm_soft_reset) {
3945 tmp = RREG32(GRBM_SOFT_RESET);
3946 tmp |= grbm_soft_reset;
3947 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3948 WREG32(GRBM_SOFT_RESET, tmp);
3949 tmp = RREG32(GRBM_SOFT_RESET);
3950
3951 udelay(50);
3952
3953 tmp &= ~grbm_soft_reset;
3954 WREG32(GRBM_SOFT_RESET, tmp);
3955 tmp = RREG32(GRBM_SOFT_RESET);
3956 }
3957
3958 if (srbm_soft_reset) {
3959 tmp = RREG32(SRBM_SOFT_RESET);
3960 tmp |= srbm_soft_reset;
3961 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3962 WREG32(SRBM_SOFT_RESET, tmp);
3963 tmp = RREG32(SRBM_SOFT_RESET);
3964
3965 udelay(50);
3966
3967 tmp &= ~srbm_soft_reset;
3968 WREG32(SRBM_SOFT_RESET, tmp);
3969 tmp = RREG32(SRBM_SOFT_RESET);
3970 }
06bc6df0
AD		 3971
3972 /* Wait a little for things to settle down */
3973 udelay(50);
3974
c476dde2 3975 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3976 udelay(50);
3977
1c534671 3978 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3979}
3980
4a5c8ea5
AD		 3981static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3982{
3983 u32 tmp, i;
3984
3985 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3986 tmp |= SPLL_BYPASS_EN;
3987 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3988
3989 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3990 tmp |= SPLL_CTLREQ_CHG;
3991 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3992
3993 for (i = 0; i < rdev->usec_timeout; i++) {
3994 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3995 break;
3996 udelay(1);
3997 }
3998
3999 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
4000 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
4001 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
4002
4003 tmp = RREG32(MPLL_CNTL_MODE);
4004 tmp &= ~MPLL_MCLK_SEL;
4005 WREG32(MPLL_CNTL_MODE, tmp);
4006}
4007
4008static void si_spll_powerdown(struct radeon_device *rdev)
4009{
4010 u32 tmp;
4011
4012 tmp = RREG32(SPLL_CNTL_MODE);
4013 tmp |= SPLL_SW_DIR_CONTROL;
4014 WREG32(SPLL_CNTL_MODE, tmp);
4015
4016 tmp = RREG32(CG_SPLL_FUNC_CNTL);
4017 tmp |= SPLL_RESET;
4018 WREG32(CG_SPLL_FUNC_CNTL, tmp);
4019
4020 tmp = RREG32(CG_SPLL_FUNC_CNTL);
4021 tmp |= SPLL_SLEEP;
4022 WREG32(CG_SPLL_FUNC_CNTL, tmp);
4023
4024 tmp = RREG32(SPLL_CNTL_MODE);
4025 tmp &= ~SPLL_SW_DIR_CONTROL;
4026 WREG32(SPLL_CNTL_MODE, tmp);
4027}
4028
4029static void si_gpu_pci_config_reset(struct radeon_device *rdev)
4030{
4031 struct evergreen_mc_save save;
4032 u32 tmp, i;
4033
4034 dev_info(rdev->dev, "GPU pci config reset\n");
4035
4036 /* disable dpm? */
4037
4038 /* disable cg/pg */
4039 si_fini_pg(rdev);
4040 si_fini_cg(rdev);
4041
4042 /* Disable CP parsing/prefetching */
4043 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4044 /* dma0 */
4045 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4046 tmp &= ~DMA_RB_ENABLE;
4047 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4048 /* dma1 */
4049 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4050 tmp &= ~DMA_RB_ENABLE;
4051 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4052 /* XXX other engines? */
4053
4054 /* halt the rlc, disable cp internal ints */
4055 si_rlc_stop(rdev);
4056
4057 udelay(50);
4058
4059 /* disable mem access */
4060 evergreen_mc_stop(rdev, &save);
4061 if (evergreen_mc_wait_for_idle(rdev)) {
4062 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4063 }
4064
4065 /* set mclk/sclk to bypass */
4066 si_set_clk_bypass_mode(rdev);
4067 /* powerdown spll */
4068 si_spll_powerdown(rdev);
4069 /* disable BM */
4070 pci_clear_master(rdev->pdev);
4071 /* reset */
4072 radeon_pci_config_reset(rdev);
4073 /* wait for asic to come out of reset */
4074 for (i = 0; i < rdev->usec_timeout; i++) {
4075 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4076 break;
4077 udelay(1);
4078 }
4079}
4080
71fe2899 4081int si_asic_reset(struct radeon_device *rdev, bool hard)
c476dde2 4082{
014bb209
AD		 4083 u32 reset_mask;
4084
71fe2899
JG		 4085 if (hard) {
4086 si_gpu_pci_config_reset(rdev);
4087 return 0;
4088 }
4089
014bb209
AD		 4090 reset_mask = si_gpu_check_soft_reset(rdev);
4091
4092 if (reset_mask)
4093 r600_set_bios_scratch_engine_hung(rdev, true);
4094
4a5c8ea5 4095 /* try soft reset */
014bb209
AD		 4096 si_gpu_soft_reset(rdev, reset_mask);
4097
4098 reset_mask = si_gpu_check_soft_reset(rdev);
4099
4a5c8ea5
AD		 4100 /* try pci config reset */
4101 if (reset_mask && radeon_hard_reset)
4102 si_gpu_pci_config_reset(rdev);
4103
4104 reset_mask = si_gpu_check_soft_reset(rdev);
4105
014bb209
AD		 4106 if (!reset_mask)
4107 r600_set_bios_scratch_engine_hung(rdev, false);
4108
4109 return 0;
c476dde2
AD		 4110}
4111
123bc183
AD		 4112/**
4113 * si_gfx_is_lockup - Check if the GFX engine is locked up
4114 *
4115 * @rdev: radeon_device pointer
4116 * @ring: radeon_ring structure holding ring information
4117 *
4118 * Check if the GFX engine is locked up.
4119 * Returns true if the engine appears to be locked up, false if not.
4120 */
4121bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4122{
4123 u32 reset_mask = si_gpu_check_soft_reset(rdev);
4124
4125 if (!(reset_mask & (RADEON_RESET_GFX |
4126 RADEON_RESET_COMPUTE |
4127 RADEON_RESET_CP))) {
ff212f25 4128 radeon_ring_lockup_update(rdev, ring);
123bc183
AD		 4129 return false;
4130 }
123bc183
AD		 4131 return radeon_ring_test_lockup(rdev, ring);
4132}
4133
d2800ee5
AD		 4134/* MC */
4135static void si_mc_program(struct radeon_device *rdev)
4136{
4137 struct evergreen_mc_save save;
4138 u32 tmp;
4139 int i, j;
4140
4141 /* Initialize HDP */
4142 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4143 WREG32((0x2c14 + j), 0x00000000);
4144 WREG32((0x2c18 + j), 0x00000000);
4145 WREG32((0x2c1c + j), 0x00000000);
4146 WREG32((0x2c20 + j), 0x00000000);
4147 WREG32((0x2c24 + j), 0x00000000);
4148 }
4149 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4150
4151 evergreen_mc_stop(rdev, &save);
4152 if (radeon_mc_wait_for_idle(rdev)) {
4153 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4154 }
5153550a
AD		 4155 if (!ASIC_IS_NODCE(rdev))
4156 /* Lockout access through VGA aperture*/
4157 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 4158 /* Update configuration */
4159 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4160 rdev->mc.vram_start >> 12);
4161 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4162 rdev->mc.vram_end >> 12);
4163 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4164 rdev->vram_scratch.gpu_addr >> 12);
4165 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4166 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4167 WREG32(MC_VM_FB_LOCATION, tmp);
4168 /* XXX double check these! */
4169 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4170 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4171 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4172 WREG32(MC_VM_AGP_BASE, 0);
4173 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4174 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4175 if (radeon_mc_wait_for_idle(rdev)) {
4176 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4177 }
4178 evergreen_mc_resume(rdev, &save);
5153550a
AD		 4179 if (!ASIC_IS_NODCE(rdev)) {
4180 /* we need to own VRAM, so turn off the VGA renderer here
4181 * to stop it overwriting our objects */
4182 rv515_vga_render_disable(rdev);
4183 }
d2800ee5
AD		 4184}
4185
1c49165d
AD		 4186void si_vram_gtt_location(struct radeon_device *rdev,
4187 struct radeon_mc *mc)
d2800ee5
AD		 4188{
4189 if (mc->mc_vram_size > 0xFFC0000000ULL) {
4190 /* leave room for at least 1024M GTT */
4191 dev_warn(rdev->dev, "limiting VRAM\n");
4192 mc->real_vram_size = 0xFFC0000000ULL;
4193 mc->mc_vram_size = 0xFFC0000000ULL;
4194 }
9ed8b1f9 4195 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 4196 rdev->mc.gtt_base_align = 0;
9ed8b1f9 4197 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 4198}
4199
4200static int si_mc_init(struct radeon_device *rdev)
4201{
4202 u32 tmp;
4203 int chansize, numchan;
4204
4205 /* Get VRAM informations */
4206 rdev->mc.vram_is_ddr = true;
4207 tmp = RREG32(MC_ARB_RAMCFG);
4208 if (tmp & CHANSIZE_OVERRIDE) {
4209 chansize = 16;
4210 } else if (tmp & CHANSIZE_MASK) {
4211 chansize = 64;
4212 } else {
4213 chansize = 32;
4214 }
4215 tmp = RREG32(MC_SHARED_CHMAP);
4216 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4217 case 0:
4218 default:
4219 numchan = 1;
4220 break;
4221 case 1:
4222 numchan = 2;
4223 break;
4224 case 2:
4225 numchan = 4;
4226 break;
4227 case 3:
4228 numchan = 8;
4229 break;
4230 case 4:
4231 numchan = 3;
4232 break;
4233 case 5:
4234 numchan = 6;
4235 break;
4236 case 6:
4237 numchan = 10;
4238 break;
4239 case 7:
4240 numchan = 12;
4241 break;
4242 case 8:
4243 numchan = 16;
4244 break;
4245 }
4246 rdev->mc.vram_width = numchan * chansize;
4247 /* Could aper size report 0 ? */
4248 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4249 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4250 /* size in MB on si */
0ca223b0
AD		 4251 tmp = RREG32(CONFIG_MEMSIZE);
4252 /* some boards may have garbage in the upper 16 bits */
4253 if (tmp & 0xffff0000) {
4254 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4255 if (tmp & 0xffff)
4256 tmp &= 0xffff;
4257 }
4258 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4259 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
d2800ee5
AD		 4260 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4261 si_vram_gtt_location(rdev, &rdev->mc);
4262 radeon_update_bandwidth_info(rdev);
4263
4264 return 0;
4265}
4266
4267/*
4268 * GART
4269 */
4270void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4271{
4272 /* flush hdp cache */
4273 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4274
4275 /* bits 0-15 are the VM contexts0-15 */
4276 WREG32(VM_INVALIDATE_REQUEST, 1);
4277}
4278
1109ca09 4279static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 4280{
4281 int r, i;
4282
4283 if (rdev->gart.robj == NULL) {
4284 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4285 return -EINVAL;
4286 }
4287 r = radeon_gart_table_vram_pin(rdev);
4288 if (r)
4289 return r;
d2800ee5
AD		 4290 /* Setup TLB control */
4291 WREG32(MC_VM_MX_L1_TLB_CNTL,
4292 (0xA << 7) |
4293 ENABLE_L1_TLB |
ec3dbbcb 4294 ENABLE_L1_FRAGMENT_PROCESSING |
d2800ee5
AD		 4295 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4296 ENABLE_ADVANCED_DRIVER_MODEL |
4297 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4298 /* Setup L2 cache */
4299 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
ec3dbbcb 4300 ENABLE_L2_FRAGMENT_PROCESSING |
d2800ee5
AD		 4301 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4302 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4303 EFFECTIVE_L2_QUEUE_SIZE(7) |
4304 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4305 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4306 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 4307 BANK_SELECT(4) |
4308 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
d2800ee5
AD		 4309 /* setup context0 */
4310 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
7c0411d2 4311 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
d2800ee5
AD		 4312 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4313 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4314 (u32)(rdev->dummy_page.addr >> 12));
4315 WREG32(VM_CONTEXT0_CNTL2, 0);
4316 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4317 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4318
4319 WREG32(0x15D4, 0);
4320 WREG32(0x15D8, 0);
4321 WREG32(0x15DC, 0);
4322
4323 /* empty context1-15 */
d2800ee5
AD		 4324 /* set vm size, must be a multiple of 4 */
4325 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
607d4806 4326 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
23d4f1f2
AD		 4327 /* Assign the pt base to something valid for now; the pts used for
4328 * the VMs are determined by the application and setup and assigned
4329 * on the fly in the vm part of radeon_gart.c
4330 */
d2800ee5
AD		 4331 for (i = 1; i < 16; i++) {
4332 if (i < 8)
4333 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
054e01d6 4334 rdev->vm_manager.saved_table_addr[i]);
d2800ee5
AD		 4335 else
4336 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
054e01d6 4337 rdev->vm_manager.saved_table_addr[i]);
d2800ee5
AD		 4338 }
4339
4340 /* enable context1-15 */
4341 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4342 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 4343 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 4344 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 4345 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
ae133a11
CK		 4346 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4347 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4348 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4349 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4350 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4351 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4352 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4353 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4354 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4355 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4356 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4357 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 4358
4359 si_pcie_gart_tlb_flush(rdev);
4360 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4361 (unsigned)(rdev->mc.gtt_size >> 20),
4362 (unsigned long long)rdev->gart.table_addr);
4363 rdev->gart.ready = true;
4364 return 0;
4365}
4366
1109ca09 4367static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5 4368{
054e01d6
CK		 4369 unsigned i;
4370
4371 for (i = 1; i < 16; ++i) {
4372 uint32_t reg;
4373 if (i < 8)
4374 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4375 else
4376 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4377 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4378 }
4379
d2800ee5
AD		 4380 /* Disable all tables */
4381 WREG32(VM_CONTEXT0_CNTL, 0);
4382 WREG32(VM_CONTEXT1_CNTL, 0);
4383 /* Setup TLB control */
4384 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4385 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4386 /* Setup L2 cache */
4387 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4388 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4389 EFFECTIVE_L2_QUEUE_SIZE(7) |
4390 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4391 WREG32(VM_L2_CNTL2, 0);
4392 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4393 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4394 radeon_gart_table_vram_unpin(rdev);
4395}
4396
1109ca09 4397static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 4398{
4399 si_pcie_gart_disable(rdev);
4400 radeon_gart_table_vram_free(rdev);
4401 radeon_gart_fini(rdev);
4402}
4403
498dd8b3
AD		 4404/* vm parser */
4405static bool si_vm_reg_valid(u32 reg)
4406{
4407 /* context regs are fine */
4408 if (reg >= 0x28000)
4409 return true;
4410
3d02b7fe
BN		 4411 /* shader regs are also fine */
4412 if (reg >= 0xB000 && reg < 0xC000)
4413 return true;
4414
498dd8b3
AD		 4415 /* check config regs */
4416 switch (reg) {
4417 case GRBM_GFX_INDEX:
f418b88a 4418 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4419 case VGT_VTX_VECT_EJECT_REG:
4420 case VGT_CACHE_INVALIDATION:
4421 case VGT_ESGS_RING_SIZE:
4422 case VGT_GSVS_RING_SIZE:
4423 case VGT_GS_VERTEX_REUSE:
4424 case VGT_PRIMITIVE_TYPE:
4425 case VGT_INDEX_TYPE:
4426 case VGT_NUM_INDICES:
4427 case VGT_NUM_INSTANCES:
4428 case VGT_TF_RING_SIZE:
4429 case VGT_HS_OFFCHIP_PARAM:
4430 case VGT_TF_MEMORY_BASE:
4431 case PA_CL_ENHANCE:
4432 case PA_SU_LINE_STIPPLE_VALUE:
4433 case PA_SC_LINE_STIPPLE_STATE:
4434 case PA_SC_ENHANCE:
4435 case SQC_CACHES:
4436 case SPI_STATIC_THREAD_MGMT_1:
4437 case SPI_STATIC_THREAD_MGMT_2:
4438 case SPI_STATIC_THREAD_MGMT_3:
4439 case SPI_PS_MAX_WAVE_ID:
4440 case SPI_CONFIG_CNTL:
4441 case SPI_CONFIG_CNTL_1:
4442 case TA_CNTL_AUX:
113d0f9d 4443 case TA_CS_BC_BASE_ADDR:
498dd8b3
AD		 4444 return true;
4445 default:
4446 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4447 return false;
4448 }
4449}
4450
4451static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4452 u32 *ib, struct radeon_cs_packet *pkt)
4453{
4454 switch (pkt->opcode) {
4455 case PACKET3_NOP:
4456 case PACKET3_SET_BASE:
4457 case PACKET3_SET_CE_DE_COUNTERS:
4458 case PACKET3_LOAD_CONST_RAM:
4459 case PACKET3_WRITE_CONST_RAM:
4460 case PACKET3_WRITE_CONST_RAM_OFFSET:
4461 case PACKET3_DUMP_CONST_RAM:
4462 case PACKET3_INCREMENT_CE_COUNTER:
4463 case PACKET3_WAIT_ON_DE_COUNTER:
4464 case PACKET3_CE_WRITE:
4465 break;
4466 default:
4467 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4468 return -EINVAL;
4469 }
4470 return 0;
4471}
4472
e5b9e750
TS		 4473static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4474{
4475 u32 start_reg, reg, i;
4476 u32 command = ib[idx + 4];
4477 u32 info = ib[idx + 1];
4478 u32 idx_value = ib[idx];
4479 if (command & PACKET3_CP_DMA_CMD_SAS) {
4480 /* src address space is register */
4481 if (((info & 0x60000000) >> 29) == 0) {
4482 start_reg = idx_value << 2;
4483 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4484 reg = start_reg;
4485 if (!si_vm_reg_valid(reg)) {
4486 DRM_ERROR("CP DMA Bad SRC register\n");
4487 return -EINVAL;
4488 }
4489 } else {
4490 for (i = 0; i < (command & 0x1fffff); i++) {
4491 reg = start_reg + (4 * i);
4492 if (!si_vm_reg_valid(reg)) {
4493 DRM_ERROR("CP DMA Bad SRC register\n");
4494 return -EINVAL;
4495 }
4496 }
4497 }
4498 }
4499 }
4500 if (command & PACKET3_CP_DMA_CMD_DAS) {
4501 /* dst address space is register */
4502 if (((info & 0x00300000) >> 20) == 0) {
4503 start_reg = ib[idx + 2];
4504 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4505 reg = start_reg;
4506 if (!si_vm_reg_valid(reg)) {
4507 DRM_ERROR("CP DMA Bad DST register\n");
4508 return -EINVAL;
4509 }
4510 } else {
4511 for (i = 0; i < (command & 0x1fffff); i++) {
4512 reg = start_reg + (4 * i);
4513 if (!si_vm_reg_valid(reg)) {
4514 DRM_ERROR("CP DMA Bad DST register\n");
4515 return -EINVAL;
4516 }
4517 }
4518 }
4519 }
4520 }
4521 return 0;
4522}
4523
498dd8b3
AD		 4524static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4525 u32 *ib, struct radeon_cs_packet *pkt)
4526{
e5b9e750 4527 int r;
498dd8b3
AD		 4528 u32 idx = pkt->idx + 1;
4529 u32 idx_value = ib[idx];
4530 u32 start_reg, end_reg, reg, i;
4531
4532 switch (pkt->opcode) {
4533 case PACKET3_NOP:
4534 case PACKET3_SET_BASE:
4535 case PACKET3_CLEAR_STATE:
4536 case PACKET3_INDEX_BUFFER_SIZE:
4537 case PACKET3_DISPATCH_DIRECT:
4538 case PACKET3_DISPATCH_INDIRECT:
4539 case PACKET3_ALLOC_GDS:
4540 case PACKET3_WRITE_GDS_RAM:
4541 case PACKET3_ATOMIC_GDS:
4542 case PACKET3_ATOMIC:
4543 case PACKET3_OCCLUSION_QUERY:
4544 case PACKET3_SET_PREDICATION:
4545 case PACKET3_COND_EXEC:
4546 case PACKET3_PRED_EXEC:
4547 case PACKET3_DRAW_INDIRECT:
4548 case PACKET3_DRAW_INDEX_INDIRECT:
4549 case PACKET3_INDEX_BASE:
4550 case PACKET3_DRAW_INDEX_2:
4551 case PACKET3_CONTEXT_CONTROL:
4552 case PACKET3_INDEX_TYPE:
4553 case PACKET3_DRAW_INDIRECT_MULTI:
4554 case PACKET3_DRAW_INDEX_AUTO:
4555 case PACKET3_DRAW_INDEX_IMMD:
4556 case PACKET3_NUM_INSTANCES:
4557 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4558 case PACKET3_STRMOUT_BUFFER_UPDATE:
4559 case PACKET3_DRAW_INDEX_OFFSET_2:
4560 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4561 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4562 case PACKET3_MPEG_INDEX:
4563 case PACKET3_WAIT_REG_MEM:
4564 case PACKET3_MEM_WRITE:
4565 case PACKET3_PFP_SYNC_ME:
4566 case PACKET3_SURFACE_SYNC:
4567 case PACKET3_EVENT_WRITE:
4568 case PACKET3_EVENT_WRITE_EOP:
4569 case PACKET3_EVENT_WRITE_EOS:
4570 case PACKET3_SET_CONTEXT_REG:
4571 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4572 case PACKET3_SET_SH_REG:
4573 case PACKET3_SET_SH_REG_OFFSET:
4574 case PACKET3_INCREMENT_DE_COUNTER:
4575 case PACKET3_WAIT_ON_CE_COUNTER:
4576 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4577 case PACKET3_ME_WRITE:
4578 break;
4579 case PACKET3_COPY_DATA:
4580 if ((idx_value & 0xf00) == 0) {
4581 reg = ib[idx + 3] * 4;
4582 if (!si_vm_reg_valid(reg))
4583 return -EINVAL;
4584 }
4585 break;
4586 case PACKET3_WRITE_DATA:
4587 if ((idx_value & 0xf00) == 0) {
4588 start_reg = ib[idx + 1] * 4;
4589 if (idx_value & 0x10000) {
4590 if (!si_vm_reg_valid(start_reg))
4591 return -EINVAL;
4592 } else {
4593 for (i = 0; i < (pkt->count - 2); i++) {
4594 reg = start_reg + (4 * i);
4595 if (!si_vm_reg_valid(reg))
4596 return -EINVAL;
4597 }
4598 }
4599 }
4600 break;
4601 case PACKET3_COND_WRITE:
4602 if (idx_value & 0x100) {
4603 reg = ib[idx + 5] * 4;
4604 if (!si_vm_reg_valid(reg))
4605 return -EINVAL;
4606 }
4607 break;
4608 case PACKET3_COPY_DW:
4609 if (idx_value & 0x2) {
4610 reg = ib[idx + 3] * 4;
4611 if (!si_vm_reg_valid(reg))
4612 return -EINVAL;
4613 }
4614 break;
4615 case PACKET3_SET_CONFIG_REG:
4616 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4617 end_reg = 4 * pkt->count + start_reg - 4;
4618 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4619 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4620 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4621 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4622 return -EINVAL;
4623 }
4624 for (i = 0; i < pkt->count; i++) {
4625 reg = start_reg + (4 * i);
4626 if (!si_vm_reg_valid(reg))
4627 return -EINVAL;
4628 }
4629 break;
5aa709be 4630 case PACKET3_CP_DMA:
e5b9e750
TS		 4631 r = si_vm_packet3_cp_dma_check(ib, idx);
4632 if (r)
4633 return r;
5aa709be 4634 break;
498dd8b3
AD		 4635 default:
4636 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4637 return -EINVAL;
4638 }
4639 return 0;
4640}
4641
4642static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4643 u32 *ib, struct radeon_cs_packet *pkt)
4644{
e5b9e750 4645 int r;
498dd8b3
AD		 4646 u32 idx = pkt->idx + 1;
4647 u32 idx_value = ib[idx];
4648 u32 start_reg, reg, i;
4649
4650 switch (pkt->opcode) {
4651 case PACKET3_NOP:
4652 case PACKET3_SET_BASE:
4653 case PACKET3_CLEAR_STATE:
4654 case PACKET3_DISPATCH_DIRECT:
4655 case PACKET3_DISPATCH_INDIRECT:
4656 case PACKET3_ALLOC_GDS:
4657 case PACKET3_WRITE_GDS_RAM:
4658 case PACKET3_ATOMIC_GDS:
4659 case PACKET3_ATOMIC:
4660 case PACKET3_OCCLUSION_QUERY:
4661 case PACKET3_SET_PREDICATION:
4662 case PACKET3_COND_EXEC:
4663 case PACKET3_PRED_EXEC:
4664 case PACKET3_CONTEXT_CONTROL:
4665 case PACKET3_STRMOUT_BUFFER_UPDATE:
4666 case PACKET3_WAIT_REG_MEM:
4667 case PACKET3_MEM_WRITE:
4668 case PACKET3_PFP_SYNC_ME:
4669 case PACKET3_SURFACE_SYNC:
4670 case PACKET3_EVENT_WRITE:
4671 case PACKET3_EVENT_WRITE_EOP:
4672 case PACKET3_EVENT_WRITE_EOS:
4673 case PACKET3_SET_CONTEXT_REG:
4674 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4675 case PACKET3_SET_SH_REG:
4676 case PACKET3_SET_SH_REG_OFFSET:
4677 case PACKET3_INCREMENT_DE_COUNTER:
4678 case PACKET3_WAIT_ON_CE_COUNTER:
4679 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4680 case PACKET3_ME_WRITE:
4681 break;
4682 case PACKET3_COPY_DATA:
4683 if ((idx_value & 0xf00) == 0) {
4684 reg = ib[idx + 3] * 4;
4685 if (!si_vm_reg_valid(reg))
4686 return -EINVAL;
4687 }
4688 break;
4689 case PACKET3_WRITE_DATA:
4690 if ((idx_value & 0xf00) == 0) {
4691 start_reg = ib[idx + 1] * 4;
4692 if (idx_value & 0x10000) {
4693 if (!si_vm_reg_valid(start_reg))
4694 return -EINVAL;
4695 } else {
4696 for (i = 0; i < (pkt->count - 2); i++) {
4697 reg = start_reg + (4 * i);
4698 if (!si_vm_reg_valid(reg))
4699 return -EINVAL;
4700 }
4701 }
4702 }
4703 break;
4704 case PACKET3_COND_WRITE:
4705 if (idx_value & 0x100) {
4706 reg = ib[idx + 5] * 4;
4707 if (!si_vm_reg_valid(reg))
4708 return -EINVAL;
4709 }
4710 break;
4711 case PACKET3_COPY_DW:
4712 if (idx_value & 0x2) {
4713 reg = ib[idx + 3] * 4;
4714 if (!si_vm_reg_valid(reg))
4715 return -EINVAL;
4716 }
4717 break;
e5b9e750
TS		 4718 case PACKET3_CP_DMA:
4719 r = si_vm_packet3_cp_dma_check(ib, idx);
4720 if (r)
4721 return r;
4722 break;
498dd8b3
AD		 4723 default:
4724 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4725 return -EINVAL;
4726 }
4727 return 0;
4728}
4729
4730int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4731{
4732 int ret = 0;
9d0223d5 4733 u32 idx = 0, i;
498dd8b3
AD		 4734 struct radeon_cs_packet pkt;
4735
4736 do {
4737 pkt.idx = idx;
4e872ae2
IH		 4738 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4739 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4740 pkt.one_reg_wr = 0;
4741 switch (pkt.type) {
4e872ae2 4742 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4743 dev_err(rdev->dev, "Packet0 not allowed!\n");
4744 ret = -EINVAL;
4745 break;
4e872ae2 4746 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4747 idx += 1;
4748 break;
4e872ae2
IH		 4749 case RADEON_PACKET_TYPE3:
4750 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4751 if (ib->is_const_ib)
4752 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4753 else {
876dc9f3 4754 switch (ib->ring) {
498dd8b3
AD		 4755 case RADEON_RING_TYPE_GFX_INDEX:
4756 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4757 break;
4758 case CAYMAN_RING_TYPE_CP1_INDEX:
4759 case CAYMAN_RING_TYPE_CP2_INDEX:
4760 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4761 break;
4762 default:
876dc9f3 4763 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4764 ret = -EINVAL;
4765 break;
4766 }
4767 }
4768 idx += pkt.count + 2;
4769 break;
4770 default:
4771 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4772 ret = -EINVAL;
4773 break;
4774 }
e1b4e722
AD		 4775 if (ret) {
4776 for (i = 0; i < ib->length_dw; i++) {
4777 if (i == idx)
4778 printk("\t0x%08x <---\n", ib->ptr[i]);
4779 else
4780 printk("\t0x%08x\n", ib->ptr[i]);
4781 }
498dd8b3 4782 break;
e1b4e722 4783 }
498dd8b3
AD		 4784 } while (idx < ib->length_dw);
4785
4786 return ret;
4787}
4788
d2800ee5
AD		 4789/*
4790 * vm
4791 */
4792int si_vm_init(struct radeon_device *rdev)
4793{
4794 /* number of VMs */
4795 rdev->vm_manager.nvm = 16;
4796 /* base offset of vram pages */
4797 rdev->vm_manager.vram_base_offset = 0;
4798
4799 return 0;
4800}
4801
4802void si_vm_fini(struct radeon_device *rdev)
4803{
4804}
4805
fbf6dc7a
AD		 4806/**
4807 * si_vm_decode_fault - print human readable fault info
4808 *
4809 * @rdev: radeon_device pointer
4810 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4811 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4812 *
4813 * Print human readable fault information (SI).
4814 */
4815static void si_vm_decode_fault(struct radeon_device *rdev,
4816 u32 status, u32 addr)
4817{
4818 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4819 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4820 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4821 char *block;
4822
4823 if (rdev->family == CHIP_TAHITI) {
4824 switch (mc_id) {
4825 case 160:
4826 case 144:
4827 case 96:
4828 case 80:
4829 case 224:
4830 case 208:
4831 case 32:
4832 case 16:
4833 block = "CB";
4834 break;
4835 case 161:
4836 case 145:
4837 case 97:
4838 case 81:
4839 case 225:
4840 case 209:
4841 case 33:
4842 case 17:
4843 block = "CB_FMASK";
4844 break;
4845 case 162:
4846 case 146:
4847 case 98:
4848 case 82:
4849 case 226:
4850 case 210:
4851 case 34:
4852 case 18:
4853 block = "CB_CMASK";
4854 break;
4855 case 163:
4856 case 147:
4857 case 99:
4858 case 83:
4859 case 227:
4860 case 211:
4861 case 35:
4862 case 19:
4863 block = "CB_IMMED";
4864 break;
4865 case 164:
4866 case 148:
4867 case 100:
4868 case 84:
4869 case 228:
4870 case 212:
4871 case 36:
4872 case 20:
4873 block = "DB";
4874 break;
4875 case 165:
4876 case 149:
4877 case 101:
4878 case 85:
4879 case 229:
4880 case 213:
4881 case 37:
4882 case 21:
4883 block = "DB_HTILE";
4884 break;
4885 case 167:
4886 case 151:
4887 case 103:
4888 case 87:
4889 case 231:
4890 case 215:
4891 case 39:
4892 case 23:
4893 block = "DB_STEN";
4894 break;
4895 case 72:
4896 case 68:
4897 case 64:
4898 case 8:
4899 case 4:
4900 case 0:
4901 case 136:
4902 case 132:
4903 case 128:
4904 case 200:
4905 case 196:
4906 case 192:
4907 block = "TC";
4908 break;
4909 case 112:
4910 case 48:
4911 block = "CP";
4912 break;
4913 case 49:
4914 case 177:
4915 case 50:
4916 case 178:
4917 block = "SH";
4918 break;
4919 case 53:
4920 case 190:
4921 block = "VGT";
4922 break;
4923 case 117:
4924 block = "IH";
4925 break;
4926 case 51:
4927 case 115:
4928 block = "RLC";
4929 break;
4930 case 119:
4931 case 183:
4932 block = "DMA0";
4933 break;
4934 case 61:
4935 block = "DMA1";
4936 break;
4937 case 248:
4938 case 120:
4939 block = "HDP";
4940 break;
4941 default:
4942 block = "unknown";
4943 break;
4944 }
4945 } else {
4946 switch (mc_id) {
4947 case 32:
4948 case 16:
4949 case 96:
4950 case 80:
4951 case 160:
4952 case 144:
4953 case 224:
4954 case 208:
4955 block = "CB";
4956 break;
4957 case 33:
4958 case 17:
4959 case 97:
4960 case 81:
4961 case 161:
4962 case 145:
4963 case 225:
4964 case 209:
4965 block = "CB_FMASK";
4966 break;
4967 case 34:
4968 case 18:
4969 case 98:
4970 case 82:
4971 case 162:
4972 case 146:
4973 case 226:
4974 case 210:
4975 block = "CB_CMASK";
4976 break;
4977 case 35:
4978 case 19:
4979 case 99:
4980 case 83:
4981 case 163:
4982 case 147:
4983 case 227:
4984 case 211:
4985 block = "CB_IMMED";
4986 break;
4987 case 36:
4988 case 20:
4989 case 100:
4990 case 84:
4991 case 164:
4992 case 148:
4993 case 228:
4994 case 212:
4995 block = "DB";
4996 break;
4997 case 37:
4998 case 21:
4999 case 101:
5000 case 85:
5001 case 165:
5002 case 149:
5003 case 229:
5004 case 213:
5005 block = "DB_HTILE";
5006 break;
5007 case 39:
5008 case 23:
5009 case 103:
5010 case 87:
5011 case 167:
5012 case 151:
5013 case 231:
5014 case 215:
5015 block = "DB_STEN";
5016 break;
5017 case 72:
5018 case 68:
5019 case 8:
5020 case 4:
5021 case 136:
5022 case 132:
5023 case 200:
5024 case 196:
5025 block = "TC";
5026 break;
5027 case 112:
5028 case 48:
5029 block = "CP";
5030 break;
5031 case 49:
5032 case 177:
5033 case 50:
5034 case 178:
5035 block = "SH";
5036 break;
5037 case 53:
5038 block = "VGT";
5039 break;
5040 case 117:
5041 block = "IH";
5042 break;
5043 case 51:
5044 case 115:
5045 block = "RLC";
5046 break;
5047 case 119:
5048 case 183:
5049 block = "DMA0";
5050 break;
5051 case 61:
5052 block = "DMA1";
5053 break;
5054 case 248:
5055 case 120:
5056 block = "HDP";
5057 break;
5058 default:
5059 block = "unknown";
5060 break;
5061 }
5062 }
5063
5064 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5065 protections, vmid, addr,
5066 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5067 block, mc_id);
5068}
5069
faffaf62
CK		 5070void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5071 unsigned vm_id, uint64_t pd_addr)
d2800ee5 5072{
76c44f2c
AD		 5073 /* write new base address */
5074 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 5075 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
76c44f2c
AD		 5076 WRITE_DATA_DST_SEL(0)));
5077
faffaf62 5078 if (vm_id < 8) {
76c44f2c 5079 radeon_ring_write(ring,
faffaf62 5080 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
ee60e29f 5081 } else {
76c44f2c 5082 radeon_ring_write(ring,
faffaf62 5083 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
ee60e29f 5084 }
76c44f2c 5085 radeon_ring_write(ring, 0);
faffaf62 5086 radeon_ring_write(ring, pd_addr >> 12);
ee60e29f 5087
d2800ee5 5088 /* flush hdp cache */
76c44f2c 5089 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5090 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
76c44f2c
AD		 5091 WRITE_DATA_DST_SEL(0)));
5092 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5093 radeon_ring_write(ring, 0);
ee60e29f
CK		 5094 radeon_ring_write(ring, 0x1);
5095
d2800ee5 5096 /* bits 0-15 are the VM contexts0-15 */
76c44f2c 5097 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5098 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
76c44f2c
AD		 5099 WRITE_DATA_DST_SEL(0)));
5100 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5101 radeon_ring_write(ring, 0);
faffaf62 5102 radeon_ring_write(ring, 1 << vm_id);
58f8cf56 5103
d474ea7e
AD		 5104 /* wait for the invalidate to complete */
5105 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5106 radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) | /* always */
5107 WAIT_REG_MEM_ENGINE(0))); /* me */
5108 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5109 radeon_ring_write(ring, 0);
5110 radeon_ring_write(ring, 0); /* ref */
5111 radeon_ring_write(ring, 0); /* mask */
5112 radeon_ring_write(ring, 0x20); /* poll interval */
5113
58f8cf56
CK		 5114 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5115 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5116 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 5117}
5118
f8f84ac5
AD		 5119/*
5120 * Power and clock gating
5121 */
5122static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5123{
5124 int i;
5125
5126 for (i = 0; i < rdev->usec_timeout; i++) {
5127 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5128 break;
5129 udelay(1);
5130 }
5131
5132 for (i = 0; i < rdev->usec_timeout; i++) {
5133 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5134 break;
5135 udelay(1);
5136 }
5137}
5138
5139static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5140 bool enable)
5141{
5142 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5143 u32 mask;
5144 int i;
5145
5146 if (enable)
5147 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5148 else
5149 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5150 WREG32(CP_INT_CNTL_RING0, tmp);
5151
5152 if (!enable) {
5153 /* read a gfx register */
5154 tmp = RREG32(DB_DEPTH_INFO);
5155
5156 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5157 for (i = 0; i < rdev->usec_timeout; i++) {
5158 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5159 break;
5160 udelay(1);
5161 }
5162 }
5163}
5164
5165static void si_set_uvd_dcm(struct radeon_device *rdev,
5166 bool sw_mode)
5167{
5168 u32 tmp, tmp2;
5169
5170 tmp = RREG32(UVD_CGC_CTRL);
5171 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5172 tmp |= DCM | CG_DT(1) | CLK_OD(4);
5173
5174 if (sw_mode) {
5175 tmp &= ~0x7ffff800;
5176 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5177 } else {
5178 tmp |= 0x7ffff800;
5179 tmp2 = 0;
5180 }
5181
5182 WREG32(UVD_CGC_CTRL, tmp);
5183 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5184}
5185
22c775ce 5186void si_init_uvd_internal_cg(struct radeon_device *rdev)
f8f84ac5
AD		 5187{
5188 bool hw_mode = true;
5189
5190 if (hw_mode) {
5191 si_set_uvd_dcm(rdev, false);
5192 } else {
5193 u32 tmp = RREG32(UVD_CGC_CTRL);
5194 tmp &= ~DCM;
5195 WREG32(UVD_CGC_CTRL, tmp);
5196 }
5197}
5198
5199static u32 si_halt_rlc(struct radeon_device *rdev)
5200{
5201 u32 data, orig;
5202
5203 orig = data = RREG32(RLC_CNTL);
5204
5205 if (data & RLC_ENABLE) {
5206 data &= ~RLC_ENABLE;
5207 WREG32(RLC_CNTL, data);
5208
5209 si_wait_for_rlc_serdes(rdev);
5210 }
5211
5212 return orig;
5213}
5214
5215static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5216{
5217 u32 tmp;
5218
5219 tmp = RREG32(RLC_CNTL);
5220 if (tmp != rlc)
5221 WREG32(RLC_CNTL, rlc);
5222}
5223
5224static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5225{
5226 u32 data, orig;
5227
5228 orig = data = RREG32(DMA_PG);
e16866ec 5229 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
f8f84ac5
AD		 5230 data |= PG_CNTL_ENABLE;
5231 else
5232 data &= ~PG_CNTL_ENABLE;
5233 if (orig != data)
5234 WREG32(DMA_PG, data);
5235}
5236
5237static void si_init_dma_pg(struct radeon_device *rdev)
5238{
5239 u32 tmp;
5240
5241 WREG32(DMA_PGFSM_WRITE, 0x00002000);
5242 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5243
5244 for (tmp = 0; tmp < 5; tmp++)
5245 WREG32(DMA_PGFSM_WRITE, 0);
5246}
5247
5248static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5249 bool enable)
5250{
5251 u32 tmp;
5252
2b19d17f 5253 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
f8f84ac5
AD		 5254 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5255 WREG32(RLC_TTOP_D, tmp);
5256
5257 tmp = RREG32(RLC_PG_CNTL);
5258 tmp |= GFX_PG_ENABLE;
5259 WREG32(RLC_PG_CNTL, tmp);
5260
5261 tmp = RREG32(RLC_AUTO_PG_CTRL);
5262 tmp |= AUTO_PG_EN;
5263 WREG32(RLC_AUTO_PG_CTRL, tmp);
5264 } else {
5265 tmp = RREG32(RLC_AUTO_PG_CTRL);
5266 tmp &= ~AUTO_PG_EN;
5267 WREG32(RLC_AUTO_PG_CTRL, tmp);
5268
5269 tmp = RREG32(DB_RENDER_CONTROL);
5270 }
5271}
5272
5273static void si_init_gfx_cgpg(struct radeon_device *rdev)
5274{
5275 u32 tmp;
5276
5277 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5278
5279 tmp = RREG32(RLC_PG_CNTL);
5280 tmp |= GFX_PG_SRC;
5281 WREG32(RLC_PG_CNTL, tmp);
5282
5283 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5284
5285 tmp = RREG32(RLC_AUTO_PG_CTRL);
5286
5287 tmp &= ~GRBM_REG_SGIT_MASK;
5288 tmp |= GRBM_REG_SGIT(0x700);
5289 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5290 WREG32(RLC_AUTO_PG_CTRL, tmp);
5291}
5292
ba19031a 5293static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
f8f84ac5
AD		 5294{
5295 u32 mask = 0, tmp, tmp1;
5296 int i;
5297
5298 si_select_se_sh(rdev, se, sh);
5299 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5300 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5301 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5302
5303 tmp &= 0xffff0000;
5304
5305 tmp |= tmp1;
5306 tmp >>= 16;
5307
5308 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5309 mask <<= 1;
5310 mask |= 1;
5311 }
5312
5313 return (~tmp) & mask;
5314}
5315
5316static void si_init_ao_cu_mask(struct radeon_device *rdev)
5317{
5318 u32 i, j, k, active_cu_number = 0;
5319 u32 mask, counter, cu_bitmap;
5320 u32 tmp = 0;
5321
5322 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5323 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5324 mask = 1;
5325 cu_bitmap = 0;
5326 counter = 0;
5327 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
ba19031a 5328 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
f8f84ac5
AD		 5329 if (counter < 2)
5330 cu_bitmap |= mask;
5331 counter++;
5332 }
5333 mask <<= 1;
5334 }
5335
5336 active_cu_number += counter;
5337 tmp |= (cu_bitmap << (i * 16 + j * 8));
5338 }
5339 }
5340
5341 WREG32(RLC_PG_AO_CU_MASK, tmp);
5342
5343 tmp = RREG32(RLC_MAX_PG_CU);
5344 tmp &= ~MAX_PU_CU_MASK;
5345 tmp |= MAX_PU_CU(active_cu_number);
5346 WREG32(RLC_MAX_PG_CU, tmp);
5347}
5348
5349static void si_enable_cgcg(struct radeon_device *rdev,
5350 bool enable)
5351{
5352 u32 data, orig, tmp;
5353
5354 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5355
e16866ec 5356 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5594a558 5357 si_enable_gui_idle_interrupt(rdev, true);
f8f84ac5 5358
f8f84ac5
AD		 5359 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5360
5361 tmp = si_halt_rlc(rdev);
5362
5363 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5364 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5365 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5366
5367 si_wait_for_rlc_serdes(rdev);
5368
5369 si_update_rlc(rdev, tmp);
5370
5371 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5372
5373 data |= CGCG_EN | CGLS_EN;
5374 } else {
5594a558
AD		 5375 si_enable_gui_idle_interrupt(rdev, false);
5376
f8f84ac5
AD		 5377 RREG32(CB_CGTT_SCLK_CTRL);
5378 RREG32(CB_CGTT_SCLK_CTRL);
5379 RREG32(CB_CGTT_SCLK_CTRL);
5380 RREG32(CB_CGTT_SCLK_CTRL);
5381
5382 data &= ~(CGCG_EN | CGLS_EN);
5383 }
5384
5385 if (orig != data)
5386 WREG32(RLC_CGCG_CGLS_CTRL, data);
5387}
5388
5389static void si_enable_mgcg(struct radeon_device *rdev,
5390 bool enable)
5391{
5392 u32 data, orig, tmp = 0;
5393
e16866ec 5394 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
f8f84ac5
AD		 5395 orig = data = RREG32(CGTS_SM_CTRL_REG);
5396 data = 0x96940200;
5397 if (orig != data)
5398 WREG32(CGTS_SM_CTRL_REG, data);
5399
e16866ec
AD		 5400 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5401 orig = data = RREG32(CP_MEM_SLP_CNTL);
5402 data |= CP_MEM_LS_EN;
5403 if (orig != data)
5404 WREG32(CP_MEM_SLP_CNTL, data);
5405 }
f8f84ac5
AD		 5406
5407 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5408 data &= 0xffffffc0;
5409 if (orig != data)
5410 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5411
5412 tmp = si_halt_rlc(rdev);
5413
5414 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5415 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5416 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5417
5418 si_update_rlc(rdev, tmp);
5419 } else {
5420 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5421 data |= 0x00000003;
5422 if (orig != data)
5423 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5424
5425 data = RREG32(CP_MEM_SLP_CNTL);
5426 if (data & CP_MEM_LS_EN) {
5427 data &= ~CP_MEM_LS_EN;
5428 WREG32(CP_MEM_SLP_CNTL, data);
5429 }
5430 orig = data = RREG32(CGTS_SM_CTRL_REG);
5431 data |= LS_OVERRIDE | OVERRIDE;
5432 if (orig != data)
5433 WREG32(CGTS_SM_CTRL_REG, data);
5434
5435 tmp = si_halt_rlc(rdev);
5436
5437 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5438 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5439 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5440
5441 si_update_rlc(rdev, tmp);
5442 }
5443}
5444
5445static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5446 bool enable)
5447{
5448 u32 orig, data, tmp;
5449
e16866ec 5450 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
f8f84ac5
AD		 5451 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5452 tmp |= 0x3fff;
5453 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5454
5455 orig = data = RREG32(UVD_CGC_CTRL);
5456 data |= DCM;
5457 if (orig != data)
5458 WREG32(UVD_CGC_CTRL, data);
5459
5460 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5461 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5462 } else {
5463 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5464 tmp &= ~0x3fff;
5465 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5466
5467 orig = data = RREG32(UVD_CGC_CTRL);
5468 data &= ~DCM;
5469 if (orig != data)
5470 WREG32(UVD_CGC_CTRL, data);
5471
5472 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5473 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5474 }
5475}
5476
5477static const u32 mc_cg_registers[] =
5478{
5479 MC_HUB_MISC_HUB_CG,
5480 MC_HUB_MISC_SIP_CG,
5481 MC_HUB_MISC_VM_CG,
5482 MC_XPB_CLK_GAT,
5483 ATC_MISC_CG,
5484 MC_CITF_MISC_WR_CG,
5485 MC_CITF_MISC_RD_CG,
5486 MC_CITF_MISC_VM_CG,
5487 VM_L2_CG,
5488};
5489
5490static void si_enable_mc_ls(struct radeon_device *rdev,
5491 bool enable)
5492{
5493 int i;
5494 u32 orig, data;
5495
5496 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5497 orig = data = RREG32(mc_cg_registers[i]);
e16866ec 5498 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
f8f84ac5
AD		 5499 data |= MC_LS_ENABLE;
5500 else
5501 data &= ~MC_LS_ENABLE;
5502 if (data != orig)
5503 WREG32(mc_cg_registers[i], data);
5504 }
5505}
5506
e16866ec
AD		 5507static void si_enable_mc_mgcg(struct radeon_device *rdev,
5508 bool enable)
f8f84ac5 5509{
e16866ec
AD		 5510 int i;
5511 u32 orig, data;
5512
5513 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5514 orig = data = RREG32(mc_cg_registers[i]);
5515 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5516 data |= MC_CG_ENABLE;
5517 else
5518 data &= ~MC_CG_ENABLE;
5519 if (data != orig)
5520 WREG32(mc_cg_registers[i], data);
f8f84ac5
AD		 5521 }
5522}
5523
e16866ec
AD		 5524static void si_enable_dma_mgcg(struct radeon_device *rdev,
5525 bool enable)
f8f84ac5 5526{
e16866ec
AD		 5527 u32 orig, data, offset;
5528 int i;
f8f84ac5 5529
e16866ec
AD		 5530 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5531 for (i = 0; i < 2; i++) {
5532 if (i == 0)
5533 offset = DMA0_REGISTER_OFFSET;
5534 else
5535 offset = DMA1_REGISTER_OFFSET;
5536 orig = data = RREG32(DMA_POWER_CNTL + offset);
5537 data &= ~MEM_POWER_OVERRIDE;
5538 if (data != orig)
5539 WREG32(DMA_POWER_CNTL + offset, data);
5540 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5541 }
5542 } else {
5543 for (i = 0; i < 2; i++) {
5544 if (i == 0)
5545 offset = DMA0_REGISTER_OFFSET;
5546 else
5547 offset = DMA1_REGISTER_OFFSET;
5548 orig = data = RREG32(DMA_POWER_CNTL + offset);
5549 data |= MEM_POWER_OVERRIDE;
5550 if (data != orig)
5551 WREG32(DMA_POWER_CNTL + offset, data);
5552
5553 orig = data = RREG32(DMA_CLK_CTRL + offset);
5554 data = 0xff000000;
5555 if (data != orig)
5556 WREG32(DMA_CLK_CTRL + offset, data);
5557 }
5558 }
f8f84ac5
AD		 5559}
5560
e16866ec
AD		 5561static void si_enable_bif_mgls(struct radeon_device *rdev,
5562 bool enable)
f8f84ac5 5563{
e16866ec 5564 u32 orig, data;
f8f84ac5 5565
e16866ec 5566 orig = data = RREG32_PCIE(PCIE_CNTL2);
f8f84ac5 5567
e16866ec
AD		 5568 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5569 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5570 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5571 else
5572 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5573 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5574
5575 if (orig != data)
5576 WREG32_PCIE(PCIE_CNTL2, data);
f8f84ac5
AD		 5577}
5578
e16866ec
AD		 5579static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5580 bool enable)
f8f84ac5 5581{
e16866ec 5582 u32 orig, data;
f8f84ac5 5583
e16866ec 5584 orig = data = RREG32(HDP_HOST_PATH_CNTL);
f8f84ac5 5585
e16866ec
AD		 5586 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5587 data &= ~CLOCK_GATING_DIS;
5588 else
5589 data |= CLOCK_GATING_DIS;
5590
5591 if (orig != data)
5592 WREG32(HDP_HOST_PATH_CNTL, data);
f8f84ac5
AD		 5593}
5594
e16866ec
AD		 5595static void si_enable_hdp_ls(struct radeon_device *rdev,
5596 bool enable)
347e7592 5597{
e16866ec 5598 u32 orig, data;
347e7592 5599
e16866ec 5600 orig = data = RREG32(HDP_MEM_POWER_LS);
347e7592 5601
e16866ec
AD		 5602 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5603 data |= HDP_LS_ENABLE;
5604 else
5605 data &= ~HDP_LS_ENABLE;
347e7592 5606
e16866ec
AD		 5607 if (orig != data)
5608 WREG32(HDP_MEM_POWER_LS, data);
347e7592
AD		 5609}
5610
68e3a092
AD		 5611static void si_update_cg(struct radeon_device *rdev,
5612 u32 block, bool enable)
347e7592 5613{
e16866ec 5614 if (block & RADEON_CG_BLOCK_GFX) {
811e4d58 5615 si_enable_gui_idle_interrupt(rdev, false);
e16866ec
AD		 5616 /* order matters! */
5617 if (enable) {
5618 si_enable_mgcg(rdev, true);
5619 si_enable_cgcg(rdev, true);
5620 } else {
5621 si_enable_cgcg(rdev, false);
5622 si_enable_mgcg(rdev, false);
347e7592 5623 }
811e4d58 5624 si_enable_gui_idle_interrupt(rdev, true);
347e7592
AD		 5625 }
5626
e16866ec
AD		 5627 if (block & RADEON_CG_BLOCK_MC) {
5628 si_enable_mc_mgcg(rdev, enable);
5629 si_enable_mc_ls(rdev, enable);
347e7592 5630 }
e16866ec
AD		 5631
5632 if (block & RADEON_CG_BLOCK_SDMA) {
5633 si_enable_dma_mgcg(rdev, enable);
347e7592
AD		 5634 }
5635
e16866ec
AD		 5636 if (block & RADEON_CG_BLOCK_BIF) {
5637 si_enable_bif_mgls(rdev, enable);
6d8cf000 5638 }
6d8cf000 5639
e16866ec
AD		 5640 if (block & RADEON_CG_BLOCK_UVD) {
5641 if (rdev->has_uvd) {
5642 si_enable_uvd_mgcg(rdev, enable);
bd8cd539
AD		 5643 }
5644 }
bd8cd539 5645
e16866ec
AD		 5646 if (block & RADEON_CG_BLOCK_HDP) {
5647 si_enable_hdp_mgcg(rdev, enable);
5648 si_enable_hdp_ls(rdev, enable);
347e7592 5649 }
e16866ec 5650}
f8f84ac5
AD		 5651
5652static void si_init_cg(struct radeon_device *rdev)
5653{
e16866ec
AD		 5654 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5655 RADEON_CG_BLOCK_MC |
5656 RADEON_CG_BLOCK_SDMA |
5657 RADEON_CG_BLOCK_BIF |
5658 RADEON_CG_BLOCK_HDP), true);
b2d70917 5659 if (rdev->has_uvd) {
e16866ec 5660 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
f8f84ac5 5661 si_init_uvd_internal_cg(rdev);
347e7592 5662 }
f8f84ac5 5663}
bd8cd539 5664
f8f84ac5
AD		 5665static void si_fini_cg(struct radeon_device *rdev)
5666{
0116e1ef 5667 if (rdev->has_uvd) {
e16866ec 5668 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
347e7592 5669 }
e16866ec
AD		 5670 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5671 RADEON_CG_BLOCK_MC |
5672 RADEON_CG_BLOCK_SDMA |
5673 RADEON_CG_BLOCK_BIF |
5674 RADEON_CG_BLOCK_HDP), false);
5675}
5676
59a82d0e
AD		 5677u32 si_get_csb_size(struct radeon_device *rdev)
5678{
5679 u32 count = 0;
5680 const struct cs_section_def *sect = NULL;
5681 const struct cs_extent_def *ext = NULL;
5682
5683 if (rdev->rlc.cs_data == NULL)
5684 return 0;
5685
5686 /* begin clear state */
5687 count += 2;
5688 /* context control state */
5689 count += 3;
5690
5691 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5692 for (ext = sect->section; ext->extent != NULL; ++ext) {
5693 if (sect->id == SECT_CONTEXT)
5694 count += 2 + ext->reg_count;
5695 else
5696 return 0;
5697 }
bd8cd539 5698 }
59a82d0e
AD		 5699 /* pa_sc_raster_config */
5700 count += 3;
5701 /* end clear state */
5702 count += 2;
5703 /* clear state */
5704 count += 2;
5705
5706 return count;
5707}
5708
5709void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5710{
5711 u32 count = 0, i;
5712 const struct cs_section_def *sect = NULL;
5713 const struct cs_extent_def *ext = NULL;
5714
5715 if (rdev->rlc.cs_data == NULL)
5716 return;
5717 if (buffer == NULL)
5718 return;
5719
6ba81e53
AD		 5720 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5721 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
59a82d0e 5722
6ba81e53
AD		 5723 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5724 buffer[count++] = cpu_to_le32(0x80000000);
5725 buffer[count++] = cpu_to_le32(0x80000000);
59a82d0e
AD		 5726
5727 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5728 for (ext = sect->section; ext->extent != NULL; ++ext) {
5729 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 5730 buffer[count++] =
5731 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5732 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
59a82d0e 5733 for (i = 0; i < ext->reg_count; i++)
6ba81e53 5734 buffer[count++] = cpu_to_le32(ext->extent[i]);
59a82d0e
AD		 5735 } else {
5736 return;
bd8cd539 5737 }
bd8cd539
AD		 5738 }
5739 }
bd8cd539 5740
6ba81e53
AD		 5741 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5742 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
59a82d0e
AD		 5743 switch (rdev->family) {
5744 case CHIP_TAHITI:
5745 case CHIP_PITCAIRN:
6ba81e53 5746 buffer[count++] = cpu_to_le32(0x2a00126a);
59a82d0e
AD		 5747 break;
5748 case CHIP_VERDE:
6ba81e53 5749 buffer[count++] = cpu_to_le32(0x0000124a);
59a82d0e
AD		 5750 break;
5751 case CHIP_OLAND:
6ba81e53 5752 buffer[count++] = cpu_to_le32(0x00000082);
59a82d0e
AD		 5753 break;
5754 case CHIP_HAINAN:
6ba81e53 5755 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5756 break;
5757 default:
6ba81e53 5758 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5759 break;
5760 }
5761
6ba81e53
AD		 5762 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5763 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
347e7592 5764
6ba81e53
AD		 5765 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5766 buffer[count++] = cpu_to_le32(0);
59a82d0e
AD		 5767}
5768
f8f84ac5
AD		 5769static void si_init_pg(struct radeon_device *rdev)
5770{
0116e1ef
AD		 5771 if (rdev->pg_flags) {
5772 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5773 si_init_dma_pg(rdev);
0116e1ef 5774 }
f8f84ac5 5775 si_init_ao_cu_mask(rdev);
2b19d17f 5776 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
0116e1ef 5777 si_init_gfx_cgpg(rdev);
aa34dba8
AD		 5778 } else {
5779 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5780 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
0116e1ef 5781 }
ca6ebb39
AD		 5782 si_enable_dma_pg(rdev, true);
5783 si_enable_gfx_cgpg(rdev, true);
f8f84ac5
AD		 5784 } else {
5785 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5786 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5787 }
5788}
5789
5790static void si_fini_pg(struct radeon_device *rdev)
5791{
0116e1ef 5792 if (rdev->pg_flags) {
ca6ebb39
AD		 5793 si_enable_dma_pg(rdev, false);
5794 si_enable_gfx_cgpg(rdev, false);
f8f84ac5 5795 }
347e7592
AD		 5796}
5797
347e7592
AD		 5798/*
5799 * RLC
5800 */
866d83de 5801void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5802{
f8f84ac5 5803 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5804
f8f84ac5
AD		 5805 tmp |= SOFT_RESET_RLC;
5806 WREG32(GRBM_SOFT_RESET, tmp);
5807 udelay(50);
5808 tmp &= ~SOFT_RESET_RLC;
5809 WREG32(GRBM_SOFT_RESET, tmp);
5810 udelay(50);
d719cef3
AD		 5811}
5812
347e7592
AD		 5813static void si_rlc_stop(struct radeon_device *rdev)
5814{
5815 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5816
5817 si_enable_gui_idle_interrupt(rdev, false);
5818
5819 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5820}
5821
5822static void si_rlc_start(struct radeon_device *rdev)
5823{
5824 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5825
5826 si_enable_gui_idle_interrupt(rdev, true);
5827
5828 udelay(50);
5829}
5830
5831static bool si_lbpw_supported(struct radeon_device *rdev)
5832{
5833 u32 tmp;
5834
5835 /* Enable LBPW only for DDR3 */
5836 tmp = RREG32(MC_SEQ_MISC0);
5837 if ((tmp & 0xF0000000) == 0xB0000000)
5838 return true;
5839 return false;
5840}
5841
5842static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5843{
5844 u32 tmp;
5845
5846 tmp = RREG32(RLC_LB_CNTL);
5847 if (enable)
5848 tmp |= LOAD_BALANCE_ENABLE;
5849 else
5850 tmp &= ~LOAD_BALANCE_ENABLE;
5851 WREG32(RLC_LB_CNTL, tmp);
5852
5853 if (!enable) {
5854 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5855 WREG32(SPI_LB_CU_MASK, 0x00ff);
5856 }
347e7592
AD		 5857}
5858
5859static int si_rlc_resume(struct radeon_device *rdev)
5860{
5861 u32 i;
347e7592
AD		 5862
5863 if (!rdev->rlc_fw)
5864 return -EINVAL;
5865
5866 si_rlc_stop(rdev);
5867
f8f84ac5
AD		 5868 si_rlc_reset(rdev);
5869
5870 si_init_pg(rdev);
5871
5872 si_init_cg(rdev);
5873
347e7592
AD		 5874 WREG32(RLC_RL_BASE, 0);
5875 WREG32(RLC_RL_SIZE, 0);
5876 WREG32(RLC_LB_CNTL, 0);
5877 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5878 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5879 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5880
347e7592
AD		 5881 WREG32(RLC_MC_CNTL, 0);
5882 WREG32(RLC_UCODE_CNTL, 0);
5883
629bd33c
AD		 5884 if (rdev->new_fw) {
5885 const struct rlc_firmware_header_v1_0 *hdr =
5886 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5887 u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5888 const __le32 *fw_data = (const __le32 *)
5889 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5890
5891 radeon_ucode_print_rlc_hdr(&hdr->header);
5892
5893 for (i = 0; i < fw_size; i++) {
5894 WREG32(RLC_UCODE_ADDR, i);
5895 WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5896 }
5897 } else {
5898 const __be32 *fw_data =
5899 (const __be32 *)rdev->rlc_fw->data;
5900 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5901 WREG32(RLC_UCODE_ADDR, i);
5902 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5903 }
347e7592
AD		 5904 }
5905 WREG32(RLC_UCODE_ADDR, 0);
5906
d719cef3
AD		 5907 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5908
347e7592
AD		 5909 si_rlc_start(rdev);
5910
5911 return 0;
5912}
5913
25a857fb
AD		 5914static void si_enable_interrupts(struct radeon_device *rdev)
5915{
5916 u32 ih_cntl = RREG32(IH_CNTL);
5917 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5918
5919 ih_cntl |= ENABLE_INTR;
5920 ih_rb_cntl |= IH_RB_ENABLE;
5921 WREG32(IH_CNTL, ih_cntl);
5922 WREG32(IH_RB_CNTL, ih_rb_cntl);
5923 rdev->ih.enabled = true;
5924}
5925
5926static void si_disable_interrupts(struct radeon_device *rdev)
5927{
5928 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5929 u32 ih_cntl = RREG32(IH_CNTL);
5930
5931 ih_rb_cntl &= ~IH_RB_ENABLE;
5932 ih_cntl &= ~ENABLE_INTR;
5933 WREG32(IH_RB_CNTL, ih_rb_cntl);
5934 WREG32(IH_CNTL, ih_cntl);
5935 /* set rptr, wptr to 0 */
5936 WREG32(IH_RB_RPTR, 0);
5937 WREG32(IH_RB_WPTR, 0);
5938 rdev->ih.enabled = false;
25a857fb
AD		 5939 rdev->ih.rptr = 0;
5940}
5941
5942static void si_disable_interrupt_state(struct radeon_device *rdev)
5943{
4cd096dd 5944 int i;
25a857fb
AD		 5945 u32 tmp;
5946
811e4d58
AD		 5947 tmp = RREG32(CP_INT_CNTL_RING0) &
5948 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5949 WREG32(CP_INT_CNTL_RING0, tmp);
25a857fb
AD		 5950 WREG32(CP_INT_CNTL_RING1, 0);
5951 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5952 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5953 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5954 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5955 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5956 WREG32(GRBM_INT_CNTL, 0);
18ad01ef 5957 WREG32(SRBM_INT_CNTL, 0);
4cd096dd
L		 5958 for (i = 0; i < rdev->num_crtc; i++)
5959 WREG32(INT_MASK + crtc_offsets[i], 0);
98990fae
L		 5960 for (i = 0; i < rdev->num_crtc; i++)
5961 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
25a857fb 5962
5153550a 5963 if (!ASIC_IS_NODCE(rdev)) {
e9a321c6 5964 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5153550a 5965
4cd096dd
L		 5966 for (i = 0; i < 6; i++)
5967 WREG32_AND(DC_HPDx_INT_CONTROL(i),
5968 DC_HPDx_INT_POLARITY);
5153550a 5969 }
25a857fb
AD		 5970}
5971
5972static int si_irq_init(struct radeon_device *rdev)
5973{
5974 int ret = 0;
5975 int rb_bufsz;
5976 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5977
5978 /* allocate ring */
5979 ret = r600_ih_ring_alloc(rdev);
5980 if (ret)
5981 return ret;
5982
5983 /* disable irqs */
5984 si_disable_interrupts(rdev);
5985
5986 /* init rlc */
5987 ret = si_rlc_resume(rdev);
5988 if (ret) {
5989 r600_ih_ring_fini(rdev);
5990 return ret;
5991 }
5992
5993 /* setup interrupt control */
62d91dd2
SB		 5994 /* set dummy read address to dummy page address */
5995 WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
25a857fb
AD		 5996 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5997 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5998 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5999 */
6000 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6001 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6002 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6003 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6004
6005 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 6006 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
25a857fb
AD		 6007
6008 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6009 IH_WPTR_OVERFLOW_CLEAR |
6010 (rb_bufsz << 1));
6011
6012 if (rdev->wb.enabled)
6013 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6014
6015 /* set the writeback address whether it's enabled or not */
6016 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6017 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6018
6019 WREG32(IH_RB_CNTL, ih_rb_cntl);
6020
6021 /* set rptr, wptr to 0 */
6022 WREG32(IH_RB_RPTR, 0);
6023 WREG32(IH_RB_WPTR, 0);
6024
6025 /* Default settings for IH_CNTL (disabled at first) */
6026 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6027 /* RPTR_REARM only works if msi's are enabled */
6028 if (rdev->msi_enabled)
6029 ih_cntl |= RPTR_REARM;
6030 WREG32(IH_CNTL, ih_cntl);
6031
6032 /* force the active interrupt state to all disabled */
6033 si_disable_interrupt_state(rdev);
6034
2099810f
DA		 6035 pci_set_master(rdev->pdev);
6036
25a857fb
AD		 6037 /* enable irqs */
6038 si_enable_interrupts(rdev);
6039
6040 return ret;
6041}
6042
4cd096dd 6043/* The order we write back each register here is important */
25a857fb
AD		 6044int si_irq_set(struct radeon_device *rdev)
6045{
4cd096dd 6046 int i;
811e4d58 6047 u32 cp_int_cntl;
25a857fb 6048 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
25a857fb 6049 u32 grbm_int_cntl = 0;
8c5fd7ef 6050 u32 dma_cntl, dma_cntl1;
a9e61410 6051 u32 thermal_int = 0;
25a857fb
AD		 6052
6053 if (!rdev->irq.installed) {
6054 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6055 return -EINVAL;
6056 }
6057 /* don't enable anything if the ih is disabled */
6058 if (!rdev->ih.enabled) {
6059 si_disable_interrupts(rdev);
6060 /* force the active interrupt state to all disabled */
6061 si_disable_interrupt_state(rdev);
6062 return 0;
6063 }
6064
811e4d58
AD		 6065 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6066 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6067
8c5fd7ef
AD		 6068 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6069 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6070
a9e61410
AD		 6071 thermal_int = RREG32(CG_THERMAL_INT) &
6072 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6073
25a857fb 6074 /* enable CP interrupts on all rings */
736fc37f 6075 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 6076 DRM_DEBUG("si_irq_set: sw int gfx\n");
6077 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6078 }
736fc37f 6079 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 6080 DRM_DEBUG("si_irq_set: sw int cp1\n");
6081 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6082 }
736fc37f 6083 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 6084 DRM_DEBUG("si_irq_set: sw int cp2\n");
6085 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6086 }
8c5fd7ef
AD		 6087 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6088 DRM_DEBUG("si_irq_set: sw int dma\n");
6089 dma_cntl |= TRAP_ENABLE;
6090 }
6091
6092 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6093 DRM_DEBUG("si_irq_set: sw int dma1\n");
6094 dma_cntl1 |= TRAP_ENABLE;
6095 }
25a857fb
AD		 6096
6097 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6098 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6099 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6100
8c5fd7ef
AD		 6101 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6102 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6103
25a857fb
AD		 6104 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6105
a9e61410
AD		 6106 if (rdev->irq.dpm_thermal) {
6107 DRM_DEBUG("dpm thermal\n");
6108 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6109 }
6110
4cd096dd
L		 6111 for (i = 0; i < rdev->num_crtc; i++) {
6112 radeon_irq_kms_set_irq_n_enabled(
6113 rdev, INT_MASK + crtc_offsets[i], VBLANK_INT_MASK,
6114 rdev->irq.crtc_vblank_int[i] ||
6115 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
25a857fb
AD		 6116 }
6117
98990fae
L		 6118 for (i = 0; i < rdev->num_crtc; i++)
6119 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
25a857fb 6120
5153550a 6121 if (!ASIC_IS_NODCE(rdev)) {
4cd096dd
L		 6122 for (i = 0; i < 6; i++) {
6123 radeon_irq_kms_set_irq_n_enabled(
6124 rdev, DC_HPDx_INT_CONTROL(i),
6125 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
6126 rdev->irq.hpd[i], "HPD", i);
6127 }
5153550a 6128 }
25a857fb 6129
a9e61410
AD		 6130 WREG32(CG_THERMAL_INT, thermal_int);
6131
0586915e
AD		 6132 /* posting read */
6133 RREG32(SRBM_STATUS);
6134
25a857fb
AD		 6135 return 0;
6136}
6137
4cd096dd 6138/* The order we write back each register here is important */
25a857fb
AD		 6139static inline void si_irq_ack(struct radeon_device *rdev)
6140{
98990fae 6141 int i, j;
4cd096dd 6142 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
98990fae 6143 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
25a857fb 6144
5153550a
AD		 6145 if (ASIC_IS_NODCE(rdev))
6146 return;
6147
98990fae 6148 for (i = 0; i < 6; i++) {
4cd096dd 6149 disp_int[i] = RREG32(si_disp_int_status[i]);
98990fae
L		 6150 if (i < rdev->num_crtc)
6151 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
25a857fb
AD		 6152 }
6153
98990fae
L		 6154 /* We write back each interrupt register in pairs of two */
6155 for (i = 0; i < rdev->num_crtc; i += 2) {
6156 for (j = i; j < (i + 2); j++) {
6157 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
6158 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
6159 GRPH_PFLIP_INT_CLEAR);
6160 }
25a857fb 6161
98990fae
L		 6162 for (j = i; j < (i + 2); j++) {
6163 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
6164 WREG32(VBLANK_STATUS + crtc_offsets[j],
6165 VBLANK_ACK);
6166 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
6167 WREG32(VLINE_STATUS + crtc_offsets[j],
6168 VLINE_ACK);
6169 }
4cd096dd
L		 6170 }
6171
6172 for (i = 0; i < 6; i++) {
6173 if (disp_int[i] & DC_HPD1_INTERRUPT)
6174 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
6175 }
6176
6177 for (i = 0; i < 6; i++) {
6178 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
6179 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
47f2467f 6180 }
25a857fb
AD		 6181}
6182
6183static void si_irq_disable(struct radeon_device *rdev)
6184{
6185 si_disable_interrupts(rdev);
6186 /* Wait and acknowledge irq */
6187 mdelay(1);
6188 si_irq_ack(rdev);
6189 si_disable_interrupt_state(rdev);
6190}
6191
6192static void si_irq_suspend(struct radeon_device *rdev)
6193{
6194 si_irq_disable(rdev);
6195 si_rlc_stop(rdev);
6196}
6197
9b136d51
AD		 6198static void si_irq_fini(struct radeon_device *rdev)
6199{
6200 si_irq_suspend(rdev);
6201 r600_ih_ring_fini(rdev);
6202}
6203
25a857fb
AD		 6204static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6205{
6206 u32 wptr, tmp;
6207
6208 if (rdev->wb.enabled)
6209 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6210 else
6211 wptr = RREG32(IH_RB_WPTR);
6212
6213 if (wptr & RB_OVERFLOW) {
11bab0ae 6214 wptr &= ~RB_OVERFLOW;
25a857fb
AD		 6215 /* When a ring buffer overflow happen start parsing interrupt
6216 * from the last not overwritten vector (wptr + 16). Hopefully
6217 * this should allow us to catchup.
6218 */
6cc2fda2
MD		 6219 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6220 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
25a857fb
AD		 6221 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6222 tmp = RREG32(IH_RB_CNTL);
6223 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6224 WREG32(IH_RB_CNTL, tmp);
6225 }
6226 return (wptr & rdev->ih.ptr_mask);
6227}
6228
6229/* SI IV Ring
6230 * Each IV ring entry is 128 bits:
6231 * [7:0] - interrupt source id
6232 * [31:8] - reserved
6233 * [59:32] - interrupt source data
6234 * [63:60] - reserved
6235 * [71:64] - RINGID
6236 * [79:72] - VMID
6237 * [127:80] - reserved
6238 */
6239int si_irq_process(struct radeon_device *rdev)
6240{
4cd096dd
L		 6241 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6242 u32 crtc_idx, hpd_idx;
6243 u32 mask;
25a857fb
AD		 6244 u32 wptr;
6245 u32 rptr;
6246 u32 src_id, src_data, ring_id;
6247 u32 ring_index;
25a857fb 6248 bool queue_hotplug = false;
47f2467f 6249 bool queue_dp = false;
a9e61410 6250 bool queue_thermal = false;
fbf6dc7a 6251 u32 status, addr;
4cd096dd 6252 const char *event_name;
25a857fb
AD		 6253
6254 if (!rdev->ih.enabled || rdev->shutdown)
6255 return IRQ_NONE;
6256
6257 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 6258
6259restart_ih:
6260 /* is somebody else already processing irqs? */
6261 if (atomic_xchg(&rdev->ih.lock, 1))
6262 return IRQ_NONE;
6263
25a857fb
AD		 6264 rptr = rdev->ih.rptr;
6265 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6266
25a857fb
AD		 6267 /* Order reading of wptr vs. reading of IH ring data */
6268 rmb();
6269
6270 /* display interrupts */
6271 si_irq_ack(rdev);
6272
25a857fb
AD		 6273 while (rptr != wptr) {
6274 /* wptr/rptr are in bytes! */
6275 ring_index = rptr / 4;
6276 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6277 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6278 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6279
6280 switch (src_id) {
6281 case 1: /* D1 vblank/vline */
25a857fb 6282 case 2: /* D2 vblank/vline */
25a857fb 6283 case 3: /* D3 vblank/vline */
25a857fb 6284 case 4: /* D4 vblank/vline */
25a857fb 6285 case 5: /* D5 vblank/vline */
4cd096dd
L		 6286 case 6: /* D6 vblank/vline */
6287 crtc_idx = src_id - 1;
07f18f0b 6288
4cd096dd
L		 6289 if (src_data == 0) { /* vblank */
6290 mask = LB_D1_VBLANK_INTERRUPT;
6291 event_name = "vblank";
6292
6293 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
6294 drm_handle_vblank(rdev->ddev, crtc_idx);
07f18f0b
MK		 6295 rdev->pm.vblank_sync = true;
6296 wake_up(&rdev->irq.vblank_queue);
25a857fb 6297 }
4cd096dd
L		 6298 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
6299 radeon_crtc_handle_vblank(rdev,
6300 crtc_idx);
6301 }
07f18f0b 6302
4cd096dd
L		 6303 } else if (src_data == 1) { /* vline */
6304 mask = LB_D1_VLINE_INTERRUPT;
6305 event_name = "vline";
6306 } else {
6307 DRM_DEBUG("Unhandled interrupt: %d %d\n",
6308 src_id, src_data);
25a857fb
AD		 6309 break;
6310 }
07f18f0b 6311
4cd096dd
L		 6312 if (!(disp_int[crtc_idx] & mask)) {
6313 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
6314 crtc_idx + 1, event_name);
6315 }
07f18f0b 6316
4cd096dd
L		 6317 disp_int[crtc_idx] &= ~mask;
6318 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
07f18f0b 6319
25a857fb 6320 break;
f5d636d2
CK		 6321 case 8: /* D1 page flip */
6322 case 10: /* D2 page flip */
6323 case 12: /* D3 page flip */
6324 case 14: /* D4 page flip */
6325 case 16: /* D5 page flip */
6326 case 18: /* D6 page flip */
6327 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 6328 if (radeon_use_pflipirq > 0)
6329 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 6330 break;
25a857fb 6331 case 42: /* HPD hotplug */
4cd096dd
L		 6332 if (src_data <= 5) {
6333 hpd_idx = src_data;
6334 mask = DC_HPD1_INTERRUPT;
07f18f0b 6335 queue_hotplug = true;
4cd096dd 6336 event_name = "HPD";
07f18f0b 6337
4cd096dd
L		 6338 } else if (src_data <= 11) {
6339 hpd_idx = src_data - 6;
6340 mask = DC_HPD1_RX_INTERRUPT;
07f18f0b 6341 queue_dp = true;
4cd096dd 6342 event_name = "HPD_RX";
07f18f0b 6343
4cd096dd
L		 6344 } else {
6345 DRM_DEBUG("Unhandled interrupt: %d %d\n",
6346 src_id, src_data);
47f2467f 6347 break;
4cd096dd 6348 }
07f18f0b 6349
4cd096dd
L		 6350 if (!(disp_int[hpd_idx] & mask))
6351 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
07f18f0b 6352
4cd096dd
L		 6353 disp_int[hpd_idx] &= ~mask;
6354 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
25a857fb 6355 break;
18ad01ef
CK		 6356 case 96:
6357 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6358 WREG32(SRBM_INT_ACK, 0x1);
6359 break;
b927e1c2
CK		 6360 case 124: /* UVD */
6361 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6362 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6363 break;
ae133a11
CK		 6364 case 146:
6365 case 147:
fbf6dc7a
AD		 6366 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6367 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
9b7d786b
CK		 6368 /* reset addr and status */
6369 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6370 if (addr == 0x0 && status == 0x0)
6371 break;
ae133a11
CK		 6372 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6373 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
fbf6dc7a 6374 addr);
ae133a11 6375 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
fbf6dc7a
AD		 6376 status);
6377 si_vm_decode_fault(rdev, status, addr);
ae133a11 6378 break;
25a857fb
AD		 6379 case 176: /* RINGID0 CP_INT */
6380 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6381 break;
6382 case 177: /* RINGID1 CP_INT */
6383 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6384 break;
6385 case 178: /* RINGID2 CP_INT */
6386 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6387 break;
6388 case 181: /* CP EOP event */
6389 DRM_DEBUG("IH: CP EOP\n");
6390 switch (ring_id) {
6391 case 0:
6392 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6393 break;
6394 case 1:
6395 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6396 break;
6397 case 2:
6398 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6399 break;
6400 }
6401 break;
8c5fd7ef
AD		 6402 case 224: /* DMA trap event */
6403 DRM_DEBUG("IH: DMA trap\n");
6404 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6405 break;
a9e61410
AD		 6406 case 230: /* thermal low to high */
6407 DRM_DEBUG("IH: thermal low to high\n");
6408 rdev->pm.dpm.thermal.high_to_low = false;
6409 queue_thermal = true;
6410 break;
6411 case 231: /* thermal high to low */
6412 DRM_DEBUG("IH: thermal high to low\n");
6413 rdev->pm.dpm.thermal.high_to_low = true;
6414 queue_thermal = true;
6415 break;
25a857fb
AD		 6416 case 233: /* GUI IDLE */
6417 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6418 break;
8c5fd7ef
AD		 6419 case 244: /* DMA trap event */
6420 DRM_DEBUG("IH: DMA1 trap\n");
6421 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6422 break;
25a857fb
AD		 6423 default:
6424 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6425 break;
6426 }
6427
6428 /* wptr/rptr are in bytes! */
6429 rptr += 16;
6430 rptr &= rdev->ih.ptr_mask;
f55e03b9 6431 WREG32(IH_RB_RPTR, rptr);
25a857fb 6432 }
47f2467f
DA		 6433 if (queue_dp)
6434 schedule_work(&rdev->dp_work);
25a857fb 6435 if (queue_hotplug)
cb5d4166 6436 schedule_delayed_work(&rdev->hotplug_work, 0);
a9e61410
AD		 6437 if (queue_thermal && rdev->pm.dpm_enabled)
6438 schedule_work(&rdev->pm.dpm.thermal.work);
25a857fb 6439 rdev->ih.rptr = rptr;
c20dc369
CK		 6440 atomic_set(&rdev->ih.lock, 0);
6441
6442 /* make sure wptr hasn't changed while processing */
6443 wptr = si_get_ih_wptr(rdev);
6444 if (wptr != rptr)
6445 goto restart_ih;
6446
25a857fb
AD		 6447 return IRQ_HANDLED;
6448}
6449
9b136d51
AD		 6450/*
6451 * startup/shutdown callbacks
6452 */
fa25c22e
JG		 6453static void si_uvd_init(struct radeon_device *rdev)
6454{
6455 int r;
6456
6457 if (!rdev->has_uvd)
6458 return;
6459
6460 r = radeon_uvd_init(rdev);
6461 if (r) {
6462 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
6463 /*
6464 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
6465 * to early fails uvd_v2_2_resume() and thus nothing happens
6466 * there. So it is pointless to try to go through that code
6467 * hence why we disable uvd here.
6468 */
43302df9 6469 rdev->has_uvd = false;
fa25c22e
JG		 6470 return;
6471 }
6472 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
6473 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
6474}
6475
6476static void si_uvd_start(struct radeon_device *rdev)
6477{
6478 int r;
6479
6480 if (!rdev->has_uvd)
6481 return;
6482
6483 r = uvd_v2_2_resume(rdev);
6484 if (r) {
6485 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
6486 goto error;
6487 }
6488 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
6489 if (r) {
6490 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
6491 goto error;
6492 }
6493 return;
6494
6495error:
6496 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6497}
6498
6499static void si_uvd_resume(struct radeon_device *rdev)
6500{
6501 struct radeon_ring *ring;
6502 int r;
6503
6504 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
6505 return;
6506
6507 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
70a033d2 6508 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
fa25c22e
JG		 6509 if (r) {
6510 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
6511 return;
6512 }
6513 r = uvd_v1_0_init(rdev);
6514 if (r) {
6515 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
6516 return;
6517 }
6518}
6519
d18dd759
JG		 6520static void si_vce_init(struct radeon_device *rdev)
6521{
6522 int r;
6523
6524 if (!rdev->has_vce)
6525 return;
6526
6527 r = radeon_vce_init(rdev);
6528 if (r) {
6529 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
6530 /*
6531 * At this point rdev->vce.vcpu_bo is NULL which trickles down
6532 * to early fails si_vce_start() and thus nothing happens
6533 * there. So it is pointless to try to go through that code
6534 * hence why we disable vce here.
6535 */
43302df9 6536 rdev->has_vce = false;
d18dd759
JG		 6537 return;
6538 }
6539 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
6540 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
6541 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
6542 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
6543}
6544
6545static void si_vce_start(struct radeon_device *rdev)
6546{
6547 int r;
6548
6549 if (!rdev->has_vce)
6550 return;
6551
6552 r = radeon_vce_resume(rdev);
6553 if (r) {
6554 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6555 goto error;
6556 }
6557 r = vce_v1_0_resume(rdev);
6558 if (r) {
6559 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6560 goto error;
6561 }
6562 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
6563 if (r) {
6564 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
6565 goto error;
6566 }
6567 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
6568 if (r) {
6569 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
6570 goto error;
6571 }
6572 return;
6573
6574error:
6575 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6576 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6577}
6578
6579static void si_vce_resume(struct radeon_device *rdev)
6580{
6581 struct radeon_ring *ring;
6582 int r;
6583
6584 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
6585 return;
6586
6587 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
6588 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6589 if (r) {
6590 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6591 return;
6592 }
6593 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
6594 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6595 if (r) {
6596 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6597 return;
6598 }
6599 r = vce_v1_0_init(rdev);
6600 if (r) {
6601 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
6602 return;
6603 }
6604}
6605
9b136d51
AD		 6606static int si_startup(struct radeon_device *rdev)
6607{
6608 struct radeon_ring *ring;
6609 int r;
6610
b9d305df
AD		 6611 /* enable pcie gen2/3 link */
6612 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6613 /* enable aspm */
6614 si_program_aspm(rdev);
b9d305df 6615
e5903d39
AD		 6616 /* scratch needs to be initialized before MC */
6617 r = r600_vram_scratch_init(rdev);
6618 if (r)
6619 return r;
6620
6fab3feb
AD		 6621 si_mc_program(rdev);
6622
6c7bccea
AD		 6623 if (!rdev->pm.dpm_enabled) {
6624 r = si_mc_load_microcode(rdev);
6625 if (r) {
6626 DRM_ERROR("Failed to load MC firmware!\n");
6627 return r;
6628 }
9b136d51
AD		 6629 }
6630
9b136d51
AD		 6631 r = si_pcie_gart_enable(rdev);
6632 if (r)
6633 return r;
6634 si_gpu_init(rdev);
6635
9b136d51 6636 /* allocate rlc buffers */
1fd11777
AD		 6637 if (rdev->family == CHIP_VERDE) {
6638 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6639 rdev->rlc.reg_list_size =
6640 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6641 }
6642 rdev->rlc.cs_data = si_cs_data;
6643 r = sumo_rlc_init(rdev);
9b136d51
AD		 6644 if (r) {
6645 DRM_ERROR("Failed to init rlc BOs!\n");
6646 return r;
6647 }
6648
6649 /* allocate wb buffer */
6650 r = radeon_wb_init(rdev);
6651 if (r)
6652 return r;
6653
6654 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6655 if (r) {
6656 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6657 return r;
6658 }
6659
6660 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6661 if (r) {
6662 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6663 return r;
6664 }
6665
6666 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6667 if (r) {
6668 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6669 return r;
6670 }
6671
8c5fd7ef
AD		 6672 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6673 if (r) {
6674 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6675 return r;
6676 }
6677
6678 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6679 if (r) {
6680 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6681 return r;
6682 }
6683
fa25c22e 6684 si_uvd_start(rdev);
d18dd759 6685 si_vce_start(rdev);
a918efab 6686
9b136d51 6687 /* Enable IRQ */
e49f3959
AH		 6688 if (!rdev->irq.installed) {
6689 r = radeon_irq_kms_init(rdev);
6690 if (r)
6691 return r;
6692 }
6693
9b136d51
AD		 6694 r = si_irq_init(rdev);
6695 if (r) {
6696 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6697 radeon_irq_kms_fini(rdev);
6698 return r;
6699 }
6700 si_irq_set(rdev);
6701
6702 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6703 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 6704 RADEON_CP_PACKET2);
9b136d51
AD		 6705 if (r)
6706 return r;
6707
6708 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6709 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
2e1e6dad 6710 RADEON_CP_PACKET2);
9b136d51
AD		 6711 if (r)
6712 return r;
6713
6714 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6715 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
2e1e6dad 6716 RADEON_CP_PACKET2);
9b136d51
AD		 6717 if (r)
6718 return r;
6719
8c5fd7ef
AD		 6720 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6721 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 6722 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6723 if (r)
6724 return r;
6725
6726 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6727 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 6728 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6729 if (r)
6730 return r;
6731
9b136d51
AD		 6732 r = si_cp_load_microcode(rdev);
6733 if (r)
6734 return r;
6735 r = si_cp_resume(rdev);
6736 if (r)
6737 return r;
6738
8c5fd7ef
AD		 6739 r = cayman_dma_resume(rdev);
6740 if (r)
6741 return r;
6742
fa25c22e 6743 si_uvd_resume(rdev);
d18dd759 6744 si_vce_resume(rdev);
a918efab 6745
2898c348
CK		 6746 r = radeon_ib_pool_init(rdev);
6747 if (r) {
6748 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6749 return r;
2898c348 6750 }
9b136d51 6751
c6105f24
CK		 6752 r = radeon_vm_manager_init(rdev);
6753 if (r) {
6754 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6755 return r;
c6105f24 6756 }
9b136d51 6757
bfc1f97d 6758 r = radeon_audio_init(rdev);
b530602f
AD		 6759 if (r)
6760 return r;
6761
9b136d51
AD		 6762 return 0;
6763}
6764
6765int si_resume(struct radeon_device *rdev)
6766{
6767 int r;
6768
6769 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6770 * posting will perform necessary task to bring back GPU into good
6771 * shape.
6772 */
6773 /* post card */
6774 atom_asic_init(rdev->mode_info.atom_context);
6775
205996c0
AD		 6776 /* init golden registers */
6777 si_init_golden_registers(rdev);
6778
bc6a6295
AD		 6779 if (rdev->pm.pm_method == PM_METHOD_DPM)
6780 radeon_pm_resume(rdev);
6c7bccea 6781
9b136d51
AD		 6782 rdev->accel_working = true;
6783 r = si_startup(rdev);
6784 if (r) {
6785 DRM_ERROR("si startup failed on resume\n");
6786 rdev->accel_working = false;
6787 return r;
6788 }
6789
6790 return r;
6791
6792}
6793
6794int si_suspend(struct radeon_device *rdev)
6795{
6c7bccea 6796 radeon_pm_suspend(rdev);
7991d665 6797 radeon_audio_fini(rdev);
fa3daf9a 6798 radeon_vm_manager_fini(rdev);
9b136d51 6799 si_cp_enable(rdev, false);
8c5fd7ef 6800 cayman_dma_stop(rdev);
1df0d523 6801 if (rdev->has_uvd) {
e409b128 6802 uvd_v1_0_fini(rdev);
1df0d523
AD		 6803 radeon_uvd_suspend(rdev);
6804 }
d18dd759
JG		 6805 if (rdev->has_vce)
6806 radeon_vce_suspend(rdev);
e16866ec
AD		 6807 si_fini_pg(rdev);
6808 si_fini_cg(rdev);
9b136d51
AD		 6809 si_irq_suspend(rdev);
6810 radeon_wb_disable(rdev);
6811 si_pcie_gart_disable(rdev);
6812 return 0;
6813}
6814
6815/* Plan is to move initialization in that function and use
6816 * helper function so that radeon_device_init pretty much
6817 * do nothing more than calling asic specific function. This
6818 * should also allow to remove a bunch of callback function
6819 * like vram_info.
6820 */
6821int si_init(struct radeon_device *rdev)
6822{
6823 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6824 int r;
6825
9b136d51
AD		 6826 /* Read BIOS */
6827 if (!radeon_get_bios(rdev)) {
6828 if (ASIC_IS_AVIVO(rdev))
6829 return -EINVAL;
6830 }
6831 /* Must be an ATOMBIOS */
6832 if (!rdev->is_atom_bios) {
6833 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6834 return -EINVAL;
6835 }
6836 r = radeon_atombios_init(rdev);
6837 if (r)
6838 return r;
6839
6840 /* Post card if necessary */
6841 if (!radeon_card_posted(rdev)) {
6842 if (!rdev->bios) {
6843 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6844 return -EINVAL;
6845 }
6846 DRM_INFO("GPU not posted. posting now...\n");
6847 atom_asic_init(rdev->mode_info.atom_context);
6848 }
205996c0
AD		 6849 /* init golden registers */
6850 si_init_golden_registers(rdev);
9b136d51
AD		 6851 /* Initialize scratch registers */
6852 si_scratch_init(rdev);
6853 /* Initialize surface registers */
6854 radeon_surface_init(rdev);
6855 /* Initialize clocks */
6856 radeon_get_clock_info(rdev->ddev);
6857
6858 /* Fence driver */
6859 r = radeon_fence_driver_init(rdev);
6860 if (r)
6861 return r;
6862
6863 /* initialize memory controller */
6864 r = si_mc_init(rdev);
6865 if (r)
6866 return r;
6867 /* Memory manager */
6868 r = radeon_bo_init(rdev);
6869 if (r)
6870 return r;
6871
01ac8794
AD		 6872 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6873 !rdev->rlc_fw || !rdev->mc_fw) {
6874 r = si_init_microcode(rdev);
6875 if (r) {
6876 DRM_ERROR("Failed to load firmware!\n");
6877 return r;
6878 }
6879 }
6880
6c7bccea
AD		 6881 /* Initialize power management */
6882 radeon_pm_init(rdev);
6883
9b136d51
AD		 6884 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6885 ring->ring_obj = NULL;
6886 r600_ring_init(rdev, ring, 1024 * 1024);
6887
6888 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6889 ring->ring_obj = NULL;
6890 r600_ring_init(rdev, ring, 1024 * 1024);
6891
6892 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6893 ring->ring_obj = NULL;
6894 r600_ring_init(rdev, ring, 1024 * 1024);
6895
8c5fd7ef
AD		 6896 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6897 ring->ring_obj = NULL;
6898 r600_ring_init(rdev, ring, 64 * 1024);
6899
6900 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6901 ring->ring_obj = NULL;
6902 r600_ring_init(rdev, ring, 64 * 1024);
6903
fa25c22e 6904 si_uvd_init(rdev);
d18dd759 6905 si_vce_init(rdev);
a918efab 6906
9b136d51
AD		 6907 rdev->ih.ring_obj = NULL;
6908 r600_ih_ring_init(rdev, 64 * 1024);
6909
6910 r = r600_pcie_gart_init(rdev);
6911 if (r)
6912 return r;
6913
9b136d51 6914 rdev->accel_working = true;
9b136d51
AD		 6915 r = si_startup(rdev);
6916 if (r) {
6917 dev_err(rdev->dev, "disabling GPU acceleration\n");
6918 si_cp_fini(rdev);
8c5fd7ef 6919 cayman_dma_fini(rdev);
9b136d51 6920 si_irq_fini(rdev);
1fd11777 6921 sumo_rlc_fini(rdev);
9b136d51 6922 radeon_wb_fini(rdev);
2898c348 6923 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6924 radeon_vm_manager_fini(rdev);
6925 radeon_irq_kms_fini(rdev);
6926 si_pcie_gart_fini(rdev);
6927 rdev->accel_working = false;
6928 }
6929
6930 /* Don't start up if the MC ucode is missing.
6931 * The default clocks and voltages before the MC ucode
6932 * is loaded are not suffient for advanced operations.
6933 */
6934 if (!rdev->mc_fw) {
6935 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6936 return -EINVAL;
6937 }
6938
6939 return 0;
6940}
6941
6942void si_fini(struct radeon_device *rdev)
6943{
6c7bccea 6944 radeon_pm_fini(rdev);
9b136d51 6945 si_cp_fini(rdev);
8c5fd7ef 6946 cayman_dma_fini(rdev);
f8f84ac5 6947 si_fini_pg(rdev);
e16866ec 6948 si_fini_cg(rdev);
9b136d51 6949 si_irq_fini(rdev);
1fd11777 6950 sumo_rlc_fini(rdev);
9b136d51
AD		 6951 radeon_wb_fini(rdev);
6952 radeon_vm_manager_fini(rdev);
2898c348 6953 radeon_ib_pool_fini(rdev);
9b136d51 6954 radeon_irq_kms_fini(rdev);
2858c00d 6955 if (rdev->has_uvd) {
e409b128 6956 uvd_v1_0_fini(rdev);
1df0d523 6957 radeon_uvd_fini(rdev);
2858c00d 6958 }
d18dd759
JG		 6959 if (rdev->has_vce)
6960 radeon_vce_fini(rdev);
9b136d51
AD		 6961 si_pcie_gart_fini(rdev);
6962 r600_vram_scratch_fini(rdev);
6963 radeon_gem_fini(rdev);
9b136d51
AD		 6964 radeon_fence_driver_fini(rdev);
6965 radeon_bo_fini(rdev);
6966 radeon_atombios_fini(rdev);
6967 kfree(rdev->bios);
6968 rdev->bios = NULL;
6969}
6970
6759a0a7 6971/**
d0418894 6972 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6973 *
6974 * @rdev: radeon_device pointer
6975 *
6976 * Fetches a GPU clock counter snapshot (SI).
6977 * Returns the 64 bit clock counter snapshot.
6978 */
d0418894 6979uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6980{
6981 uint64_t clock;
6982
6983 mutex_lock(&rdev->gpu_clock_mutex);
6984 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6985 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
3cf8bb1a 6986 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6759a0a7
MO		 6987 mutex_unlock(&rdev->gpu_clock_mutex);
6988 return clock;
6989}
2539eb02 6990
2539eb02
CK		 6991int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6992{
facd112d 6993 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6994 int r;
6995
4ed10835
CK		 6996 /* bypass vclk and dclk with bclk */
6997 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6998 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6999 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7000
7001 /* put PLL in bypass mode */
7002 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7003
7004 if (!vclk || !dclk) {
a17d4996 7005 /* keep the Bypass mode */
4ed10835
CK		 7006 return 0;
7007 }
7008
facd112d
CK		 7009 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7010 16384, 0x03FFFFFF, 0, 128, 5,
7011 &fb_div, &vclk_div, &dclk_div);
7012 if (r)
7013 return r;
2539eb02
CK		 7014
7015 /* set RESET_ANTI_MUX to 0 */
7016 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7017
7018 /* set VCO_MODE to 1 */
7019 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7020
a17d4996 7021 /* disable sleep mode */
2539eb02
CK		 7022 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7023
7024 /* deassert UPLL_RESET */
7025 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7026
7027 mdelay(1);
7028
facd112d 7029 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 7030 if (r)
7031 return r;
7032
7033 /* assert UPLL_RESET again */
7034 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7035
7036 /* disable spread spectrum. */
7037 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7038
7039 /* set feedback divider */
facd112d 7040 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 7041
7042 /* set ref divider to 0 */
7043 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7044
facd112d 7045 if (fb_div < 307200)
2539eb02
CK		 7046 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7047 else
7048 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7049
7050 /* set PDIV_A and PDIV_B */
7051 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 7052 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 7053 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7054
7055 /* give the PLL some time to settle */
7056 mdelay(15);
7057
7058 /* deassert PLL_RESET */
7059 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7060
7061 mdelay(15);
7062
7063 /* switch from bypass mode to normal mode */
7064 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7065
facd112d 7066 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 7067 if (r)
7068 return r;
7069
7070 /* switch VCLK and DCLK selection */
7071 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7072 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7073 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7074
7075 mdelay(100);
7076
7077 return 0;
7078}
b9d305df
AD		 7079
7080static void si_pcie_gen3_enable(struct radeon_device *rdev)
7081{
7082 struct pci_dev *root = rdev->pdev->bus->self;
5f152a57 7083 enum pci_bus_speed speed_cap;
5f152a57
AD		 7084 u32 speed_cntl, current_data_rate;
7085 int i;
b9d305df
AD		 7086 u16 tmp16;
7087
0bd252de
AW		 7088 if (pci_is_root_bus(rdev->pdev->bus))
7089 return;
7090
b9d305df
AD		 7091 if (radeon_pcie_gen2 == 0)
7092 return;
7093
7094 if (rdev->flags & RADEON_IS_IGP)
7095 return;
7096
7097 if (!(rdev->flags & RADEON_IS_PCIE))
7098 return;
7099
5f152a57
AD		 7100 speed_cap = pcie_get_speed_cap(root);
7101 if (speed_cap == PCI_SPEED_UNKNOWN)
b9d305df
AD		 7102 return;
7103
5f152a57
AD		 7104 if ((speed_cap != PCIE_SPEED_8_0GT) &&
7105 (speed_cap != PCIE_SPEED_5_0GT))
b9d305df
AD		 7106 return;
7107
7108 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7109 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7110 LC_CURRENT_DATA_RATE_SHIFT;
5f152a57 7111 if (speed_cap == PCIE_SPEED_8_0GT) {
b9d305df
AD		 7112 if (current_data_rate == 2) {
7113 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7114 return;
7115 }
7116 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
5f152a57 7117 } else if (speed_cap == PCIE_SPEED_5_0GT) {
b9d305df
AD		 7118 if (current_data_rate == 1) {
7119 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7120 return;
7121 }
7122 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7123 }
7124
3d581b11 7125 if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev))
b9d305df
AD		 7126 return;
7127
5f152a57 7128 if (speed_cap == PCIE_SPEED_8_0GT) {
b9d305df
AD		 7129 /* re-try equalization if gen3 is not already enabled */
7130 if (current_data_rate != 2) {
7131 u16 bridge_cfg, gpu_cfg;
7132 u16 bridge_cfg2, gpu_cfg2;
7133 u32 max_lw, current_lw, tmp;
7134
3d581b11
FL		 7135 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
7136 &bridge_cfg);
7137 pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL,
7138 &gpu_cfg);
b9d305df
AD		 7139
7140 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
3d581b11 7141 pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
b9d305df
AD		 7142
7143 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
3d581b11
FL		 7144 pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL,
7145 tmp16);
b9d305df
AD		 7146
7147 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7148 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7149 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7150
7151 if (current_lw < max_lw) {
7152 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7153 if (tmp & LC_RENEGOTIATION_SUPPORT) {
7154 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7155 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7156 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7157 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7158 }
7159 }
7160
7161 for (i = 0; i < 10; i++) {
7162 /* check status */
3d581b11
FL		 7163 pcie_capability_read_word(rdev->pdev,
7164 PCI_EXP_DEVSTA,
7165 &tmp16);
b9d305df
AD		 7166 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7167 break;
7168
3d581b11
FL		 7169 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
7170 &bridge_cfg);
7171 pcie_capability_read_word(rdev->pdev,
7172 PCI_EXP_LNKCTL,
7173 &gpu_cfg);
b9d305df 7174
3d581b11
FL		 7175 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
7176 &bridge_cfg2);
7177 pcie_capability_read_word(rdev->pdev,
7178 PCI_EXP_LNKCTL2,
7179 &gpu_cfg2);
b9d305df
AD		 7180
7181 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7182 tmp |= LC_SET_QUIESCE;
7183 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7184
7185 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7186 tmp |= LC_REDO_EQ;
7187 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7188
2f2debb5 7189 msleep(100);
b9d305df
AD		 7190
7191 /* linkctl */
3d581b11
FL		 7192 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
7193 &tmp16);
b9d305df
AD		 7194 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7195 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
3d581b11
FL		 7196 pcie_capability_write_word(root,
7197 PCI_EXP_LNKCTL,
7198 tmp16);
b9d305df 7199
3d581b11
FL		 7200 pcie_capability_read_word(rdev->pdev,
7201 PCI_EXP_LNKCTL,
7202 &tmp16);
b9d305df
AD		 7203 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7204 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
3d581b11
FL		 7205 pcie_capability_write_word(rdev->pdev,
7206 PCI_EXP_LNKCTL,
7207 tmp16);
b9d305df
AD		 7208
7209 /* linkctl2 */
3d581b11
FL		 7210 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
7211 &tmp16);
ca56f99c
BH		 7212 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
7213 PCI_EXP_LNKCTL2_TX_MARGIN);
7214 tmp16 |= (bridge_cfg2 &
7215 (PCI_EXP_LNKCTL2_ENTER_COMP |
7216 PCI_EXP_LNKCTL2_TX_MARGIN));
3d581b11
FL		 7217 pcie_capability_write_word(root,
7218 PCI_EXP_LNKCTL2,
7219 tmp16);
b9d305df 7220
3d581b11
FL		 7221 pcie_capability_read_word(rdev->pdev,
7222 PCI_EXP_LNKCTL2,
7223 &tmp16);
ca56f99c
BH		 7224 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
7225 PCI_EXP_LNKCTL2_TX_MARGIN);
7226 tmp16 |= (gpu_cfg2 &
7227 (PCI_EXP_LNKCTL2_ENTER_COMP |
7228 PCI_EXP_LNKCTL2_TX_MARGIN));
3d581b11
FL		 7229 pcie_capability_write_word(rdev->pdev,
7230 PCI_EXP_LNKCTL2,
7231 tmp16);
b9d305df
AD		 7232
7233 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7234 tmp &= ~LC_SET_QUIESCE;
7235 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7236 }
7237 }
7238 }
7239
7240 /* set the link speed */
7241 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7242 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7243 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7244
3d581b11 7245 pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL2, &tmp16);
ca56f99c 7246 tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
5f152a57 7247 if (speed_cap == PCIE_SPEED_8_0GT)
ca56f99c 7248 tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
5f152a57 7249 else if (speed_cap == PCIE_SPEED_5_0GT)
ca56f99c 7250 tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
b9d305df 7251 else
ca56f99c 7252 tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
3d581b11 7253 pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL2, tmp16);
b9d305df
AD		 7254
7255 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7256 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7257 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7258
7259 for (i = 0; i < rdev->usec_timeout; i++) {
7260 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7261 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7262 break;
7263 udelay(1);
7264 }
7265}
7266
e0bcf165
AD		 7267static void si_program_aspm(struct radeon_device *rdev)
7268{
7269 u32 data, orig;
7270 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7271 bool disable_clkreq = false;
7272
1294d4a3
AD		 7273 if (radeon_aspm == 0)
7274 return;
7275
e0bcf165
AD		 7276 if (!(rdev->flags & RADEON_IS_PCIE))
7277 return;
7278
7279 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7280 data &= ~LC_XMIT_N_FTS_MASK;
7281 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7282 if (orig != data)
7283 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7284
7285 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7286 data |= LC_GO_TO_RECOVERY;
7287 if (orig != data)
7288 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7289
7290 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7291 data |= P_IGNORE_EDB_ERR;
7292 if (orig != data)
7293 WREG32_PCIE(PCIE_P_CNTL, data);
7294
7295 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7296 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7297 data |= LC_PMI_TO_L1_DIS;
7298 if (!disable_l0s)
7299 data |= LC_L0S_INACTIVITY(7);
7300
7301 if (!disable_l1) {
7302 data |= LC_L1_INACTIVITY(7);
7303 data &= ~LC_PMI_TO_L1_DIS;
7304 if (orig != data)
7305 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7306
7307 if (!disable_plloff_in_l1) {
7308 bool clk_req_support;
7309
7310 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7311 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7312 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7313 if (orig != data)
7314 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7315
7316 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7317 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7318 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7319 if (orig != data)
7320 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7321
7322 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7323 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7324 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7325 if (orig != data)
7326 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7327
7328 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7329 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7330 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7331 if (orig != data)
7332 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7333
7334 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7335 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7336 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7337 if (orig != data)
7338 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7339
7340 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7341 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7342 if (orig != data)
7343 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7344
7345 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7346 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7347 if (orig != data)
7348 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7349
7350 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7351 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7352 if (orig != data)
7353 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7354
7355 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7356 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7357 if (orig != data)
7358 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7359
7360 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7361 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7362 if (orig != data)
7363 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7364
7365 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7366 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7367 if (orig != data)
7368 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7369
7370 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7371 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7372 if (orig != data)
7373 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7374 }
7375 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7376 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7377 data |= LC_DYN_LANES_PWR_STATE(3);
7378 if (orig != data)
7379 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7380
7381 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7382 data &= ~LS2_EXIT_TIME_MASK;
7383 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7384 data |= LS2_EXIT_TIME(5);
7385 if (orig != data)
7386 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7387
7388 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7389 data &= ~LS2_EXIT_TIME_MASK;
7390 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7391 data |= LS2_EXIT_TIME(5);
7392 if (orig != data)
7393 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7394
0bd252de
AW		 7395 if (!disable_clkreq &&
7396 !pci_is_root_bus(rdev->pdev->bus)) {
e0bcf165
AD		 7397 struct pci_dev *root = rdev->pdev->bus->self;
7398 u32 lnkcap;
7399
7400 clk_req_support = false;
7401 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7402 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7403 clk_req_support = true;
7404 } else {
7405 clk_req_support = false;
7406 }
7407
7408 if (clk_req_support) {
7409 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7410 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7411 if (orig != data)
7412 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7413
7414 orig = data = RREG32(THM_CLK_CNTL);
7415 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7416 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7417 if (orig != data)
7418 WREG32(THM_CLK_CNTL, data);
7419
7420 orig = data = RREG32(MISC_CLK_CNTL);
7421 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7422 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7423 if (orig != data)
7424 WREG32(MISC_CLK_CNTL, data);
7425
7426 orig = data = RREG32(CG_CLKPIN_CNTL);
7427 data &= ~BCLK_AS_XCLK;
7428 if (orig != data)
7429 WREG32(CG_CLKPIN_CNTL, data);
7430
7431 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7432 data &= ~FORCE_BIF_REFCLK_EN;
7433 if (orig != data)
7434 WREG32(CG_CLKPIN_CNTL_2, data);
7435
7436 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7437 data &= ~MPLL_CLKOUT_SEL_MASK;
7438 data |= MPLL_CLKOUT_SEL(4);
7439 if (orig != data)
7440 WREG32(MPLL_BYPASSCLK_SEL, data);
7441
7442 orig = data = RREG32(SPLL_CNTL_MODE);
7443 data &= ~SPLL_REFCLK_SEL_MASK;
7444 if (orig != data)
7445 WREG32(SPLL_CNTL_MODE, data);
7446 }
7447 }
7448 } else {
7449 if (orig != data)
7450 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7451 }
7452
7453 orig = data = RREG32_PCIE(PCIE_CNTL2);
7454 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7455 if (orig != data)
7456 WREG32_PCIE(PCIE_CNTL2, data);
7457
7458 if (!disable_l0s) {
7459 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7460 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7461 data = RREG32_PCIE(PCIE_LC_STATUS1);
7462 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7463 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7464 data &= ~LC_L0S_INACTIVITY_MASK;
7465 if (orig != data)
7466 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7467 }
7468 }
7469 }
7470}
b7af630c 7471
22e5808e 7472static int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
b7af630c 7473{
3cf8bb1a 7474 unsigned i;
b7af630c 7475
3cf8bb1a
JG		 7476 /* make sure VCEPLL_CTLREQ is deasserted */
7477 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
b7af630c 7478
3cf8bb1a 7479 mdelay(10);
b7af630c 7480
3cf8bb1a
JG		 7481 /* assert UPLL_CTLREQ */
7482 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
b7af630c 7483
3cf8bb1a
JG		 7484 /* wait for CTLACK and CTLACK2 to get asserted */
7485 for (i = 0; i < 100; ++i) {
7486 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7487 if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7488 break;
7489 mdelay(10);
7490 }
b7af630c 7491
3cf8bb1a
JG		 7492 /* deassert UPLL_CTLREQ */
7493 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
b7af630c 7494
3cf8bb1a
JG		 7495 if (i == 100) {
7496 DRM_ERROR("Timeout setting UVD clocks!\n");
7497 return -ETIMEDOUT;
7498 }
b7af630c 7499
3cf8bb1a 7500 return 0;
b7af630c
CK		 7501}
7502
7503int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7504{
7505 unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7506 int r;
7507
7508 /* bypass evclk and ecclk with bclk */
7509 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7510 EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7511 ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7512
7513 /* put PLL in bypass mode */
7514 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7515 ~VCEPLL_BYPASS_EN_MASK);
7516
7517 if (!evclk || !ecclk) {
7518 /* keep the Bypass mode, put PLL to sleep */
7519 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7520 ~VCEPLL_SLEEP_MASK);
7521 return 0;
7522 }
7523
7524 r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7525 16384, 0x03FFFFFF, 0, 128, 5,
7526 &fb_div, &evclk_div, &ecclk_div);
7527 if (r)
7528 return r;
7529
7530 /* set RESET_ANTI_MUX to 0 */
7531 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7532
7533 /* set VCO_MODE to 1 */
7534 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7535 ~VCEPLL_VCO_MODE_MASK);
7536
7537 /* toggle VCEPLL_SLEEP to 1 then back to 0 */
7538 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7539 ~VCEPLL_SLEEP_MASK);
7540 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7541
7542 /* deassert VCEPLL_RESET */
7543 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7544
7545 mdelay(1);
7546
7547 r = si_vce_send_vcepll_ctlreq(rdev);
7548 if (r)
7549 return r;
7550
7551 /* assert VCEPLL_RESET again */
7552 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7553
7554 /* disable spread spectrum. */
7555 WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7556
7557 /* set feedback divider */
7558 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7559
7560 /* set ref divider to 0 */
7561 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7562
7563 /* set PDIV_A and PDIV_B */
7564 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7565 VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7566 ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7567
7568 /* give the PLL some time to settle */
7569 mdelay(15);
7570
7571 /* deassert PLL_RESET */
7572 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7573
7574 mdelay(15);
7575
7576 /* switch from bypass mode to normal mode */
7577 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7578
7579 r = si_vce_send_vcepll_ctlreq(rdev);
7580 if (r)
7581 return r;
7582
7583 /* switch VCLK and DCLK selection */
7584 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7585 EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7586 ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7587
7588 mdelay(100);
7589
7590 return 0;
7591}
Linux 6.x block layer and io_uring tree(s)