projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'mm-stable-2024-05-24-11-49' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
c182615f 24
0f0de06c 25#include <linux/firmware.h>
0f0de06c 26#include <linux/module.h>
2ef79416
TZ		 27#include <linux/pci.h>
28#include <linux/slab.h>
c182615f 29
c182615f 30#include <drm/drm_vblank.h>
760285e7 31#include <drm/radeon_drm.h>
c182615f 32
43b3cd99 33#include "atom.h"
bd8cd539 34#include "clearstate_si.h"
4fe1999e 35#include "evergreen.h"
eb388a88 36#include "r600.h"
c182615f
SR		 37#include "radeon.h"
38#include "radeon_asic.h"
39#include "radeon_audio.h"
a0ceada6 40#include "radeon_ucode.h"
c182615f 41#include "si_blit_shaders.h"
d93a3c27 42#include "si.h"
c182615f 43#include "sid.h"
43b3cd99 44
0f0de06c
AD		 45
46MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
47MODULE_FIRMWARE("radeon/TAHITI_me.bin");
48MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
49MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
1ebe9280 50MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
0f0de06c 51MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
a9e61410 52MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
629bd33c
AD		 53
54MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
55MODULE_FIRMWARE("radeon/tahiti_me.bin");
56MODULE_FIRMWARE("radeon/tahiti_ce.bin");
57MODULE_FIRMWARE("radeon/tahiti_mc.bin");
58MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
59MODULE_FIRMWARE("radeon/tahiti_smc.bin");
60
0f0de06c
AD		 61MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
62MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
63MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
64MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
1ebe9280 65MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
0f0de06c 66MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
a9e61410 67MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
629bd33c
AD		 68
69MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
70MODULE_FIRMWARE("radeon/pitcairn_me.bin");
71MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
72MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
73MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
74MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
861c7fde 75MODULE_FIRMWARE("radeon/pitcairn_k_smc.bin");
629bd33c 76
0f0de06c
AD		 77MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
78MODULE_FIRMWARE("radeon/VERDE_me.bin");
79MODULE_FIRMWARE("radeon/VERDE_ce.bin");
80MODULE_FIRMWARE("radeon/VERDE_mc.bin");
1ebe9280 81MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
0f0de06c 82MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
a9e61410 83MODULE_FIRMWARE("radeon/VERDE_smc.bin");
629bd33c
AD		 84
85MODULE_FIRMWARE("radeon/verde_pfp.bin");
86MODULE_FIRMWARE("radeon/verde_me.bin");
87MODULE_FIRMWARE("radeon/verde_ce.bin");
88MODULE_FIRMWARE("radeon/verde_mc.bin");
89MODULE_FIRMWARE("radeon/verde_rlc.bin");
90MODULE_FIRMWARE("radeon/verde_smc.bin");
861c7fde 91MODULE_FIRMWARE("radeon/verde_k_smc.bin");
629bd33c 92
bcc7f5d2
AD		 93MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
94MODULE_FIRMWARE("radeon/OLAND_me.bin");
95MODULE_FIRMWARE("radeon/OLAND_ce.bin");
96MODULE_FIRMWARE("radeon/OLAND_mc.bin");
1ebe9280 97MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
bcc7f5d2 98MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
a9e61410 99MODULE_FIRMWARE("radeon/OLAND_smc.bin");
629bd33c
AD		 100
101MODULE_FIRMWARE("radeon/oland_pfp.bin");
102MODULE_FIRMWARE("radeon/oland_me.bin");
103MODULE_FIRMWARE("radeon/oland_ce.bin");
104MODULE_FIRMWARE("radeon/oland_mc.bin");
105MODULE_FIRMWARE("radeon/oland_rlc.bin");
106MODULE_FIRMWARE("radeon/oland_smc.bin");
861c7fde 107MODULE_FIRMWARE("radeon/oland_k_smc.bin");
629bd33c 108
c04c00b4
AD		 109MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
110MODULE_FIRMWARE("radeon/HAINAN_me.bin");
111MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
112MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
1ebe9280 113MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
c04c00b4 114MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
a9e61410 115MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
0f0de06c 116
629bd33c
AD		 117MODULE_FIRMWARE("radeon/hainan_pfp.bin");
118MODULE_FIRMWARE("radeon/hainan_me.bin");
119MODULE_FIRMWARE("radeon/hainan_ce.bin");
120MODULE_FIRMWARE("radeon/hainan_mc.bin");
121MODULE_FIRMWARE("radeon/hainan_rlc.bin");
122MODULE_FIRMWARE("radeon/hainan_smc.bin");
861c7fde 123MODULE_FIRMWARE("radeon/hainan_k_smc.bin");
4e6e98b1 124MODULE_FIRMWARE("radeon/banks_k_2_smc.bin");
629bd33c 125
ef736d39
AD		 126MODULE_FIRMWARE("radeon/si58_mc.bin");
127
65fcf668 128static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
b9d305df 129static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 130static void si_program_aspm(struct radeon_device *rdev);
1fd11777
AD		 131extern void sumo_rlc_fini(struct radeon_device *rdev);
132extern int sumo_rlc_init(struct radeon_device *rdev);
811e4d58
AD		 133static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
134 bool enable);
4a5c8ea5
AD		 135static void si_init_pg(struct radeon_device *rdev);
136static void si_init_cg(struct radeon_device *rdev);
a6f4ae8d
AD		 137static void si_fini_pg(struct radeon_device *rdev);
138static void si_fini_cg(struct radeon_device *rdev);
139static void si_rlc_stop(struct radeon_device *rdev);
0a96d72b 140
ce136e15 141static const u32 crtc_offsets[] = {
4cd096dd
L		 142 EVERGREEN_CRTC0_REGISTER_OFFSET,
143 EVERGREEN_CRTC1_REGISTER_OFFSET,
144 EVERGREEN_CRTC2_REGISTER_OFFSET,
145 EVERGREEN_CRTC3_REGISTER_OFFSET,
146 EVERGREEN_CRTC4_REGISTER_OFFSET,
147 EVERGREEN_CRTC5_REGISTER_OFFSET
148};
149
ce136e15 150static const u32 si_disp_int_status[] = {
4cd096dd
L		 151 DISP_INTERRUPT_STATUS,
152 DISP_INTERRUPT_STATUS_CONTINUE,
153 DISP_INTERRUPT_STATUS_CONTINUE2,
154 DISP_INTERRUPT_STATUS_CONTINUE3,
155 DISP_INTERRUPT_STATUS_CONTINUE4,
156 DISP_INTERRUPT_STATUS_CONTINUE5
157};
158
159#define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
160#define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
161#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
162
ce136e15 163static const u32 verde_rlc_save_restore_register_list[] = {
6d8cf000
AD		 164 (0x8000 << 16) | (0x98f4 >> 2),
165 0x00000000,
166 (0x8040 << 16) | (0x98f4 >> 2),
167 0x00000000,
168 (0x8000 << 16) | (0xe80 >> 2),
169 0x00000000,
170 (0x8040 << 16) | (0xe80 >> 2),
171 0x00000000,
172 (0x8000 << 16) | (0x89bc >> 2),
173 0x00000000,
174 (0x8040 << 16) | (0x89bc >> 2),
175 0x00000000,
176 (0x8000 << 16) | (0x8c1c >> 2),
177 0x00000000,
178 (0x8040 << 16) | (0x8c1c >> 2),
179 0x00000000,
180 (0x9c00 << 16) | (0x98f0 >> 2),
181 0x00000000,
182 (0x9c00 << 16) | (0xe7c >> 2),
183 0x00000000,
184 (0x8000 << 16) | (0x9148 >> 2),
185 0x00000000,
186 (0x8040 << 16) | (0x9148 >> 2),
187 0x00000000,
188 (0x9c00 << 16) | (0x9150 >> 2),
189 0x00000000,
190 (0x9c00 << 16) | (0x897c >> 2),
191 0x00000000,
192 (0x9c00 << 16) | (0x8d8c >> 2),
193 0x00000000,
194 (0x9c00 << 16) | (0xac54 >> 2),
195 0X00000000,
196 0x3,
197 (0x9c00 << 16) | (0x98f8 >> 2),
198 0x00000000,
199 (0x9c00 << 16) | (0x9910 >> 2),
200 0x00000000,
201 (0x9c00 << 16) | (0x9914 >> 2),
202 0x00000000,
203 (0x9c00 << 16) | (0x9918 >> 2),
204 0x00000000,
205 (0x9c00 << 16) | (0x991c >> 2),
206 0x00000000,
207 (0x9c00 << 16) | (0x9920 >> 2),
208 0x00000000,
209 (0x9c00 << 16) | (0x9924 >> 2),
210 0x00000000,
211 (0x9c00 << 16) | (0x9928 >> 2),
212 0x00000000,
213 (0x9c00 << 16) | (0x992c >> 2),
214 0x00000000,
215 (0x9c00 << 16) | (0x9930 >> 2),
216 0x00000000,
217 (0x9c00 << 16) | (0x9934 >> 2),
218 0x00000000,
219 (0x9c00 << 16) | (0x9938 >> 2),
220 0x00000000,
221 (0x9c00 << 16) | (0x993c >> 2),
222 0x00000000,
223 (0x9c00 << 16) | (0x9940 >> 2),
224 0x00000000,
225 (0x9c00 << 16) | (0x9944 >> 2),
226 0x00000000,
227 (0x9c00 << 16) | (0x9948 >> 2),
228 0x00000000,
229 (0x9c00 << 16) | (0x994c >> 2),
230 0x00000000,
231 (0x9c00 << 16) | (0x9950 >> 2),
232 0x00000000,
233 (0x9c00 << 16) | (0x9954 >> 2),
234 0x00000000,
235 (0x9c00 << 16) | (0x9958 >> 2),
236 0x00000000,
237 (0x9c00 << 16) | (0x995c >> 2),
238 0x00000000,
239 (0x9c00 << 16) | (0x9960 >> 2),
240 0x00000000,
241 (0x9c00 << 16) | (0x9964 >> 2),
242 0x00000000,
243 (0x9c00 << 16) | (0x9968 >> 2),
244 0x00000000,
245 (0x9c00 << 16) | (0x996c >> 2),
246 0x00000000,
247 (0x9c00 << 16) | (0x9970 >> 2),
248 0x00000000,
249 (0x9c00 << 16) | (0x9974 >> 2),
250 0x00000000,
251 (0x9c00 << 16) | (0x9978 >> 2),
252 0x00000000,
253 (0x9c00 << 16) | (0x997c >> 2),
254 0x00000000,
255 (0x9c00 << 16) | (0x9980 >> 2),
256 0x00000000,
257 (0x9c00 << 16) | (0x9984 >> 2),
258 0x00000000,
259 (0x9c00 << 16) | (0x9988 >> 2),
260 0x00000000,
261 (0x9c00 << 16) | (0x998c >> 2),
262 0x00000000,
263 (0x9c00 << 16) | (0x8c00 >> 2),
264 0x00000000,
265 (0x9c00 << 16) | (0x8c14 >> 2),
266 0x00000000,
267 (0x9c00 << 16) | (0x8c04 >> 2),
268 0x00000000,
269 (0x9c00 << 16) | (0x8c08 >> 2),
270 0x00000000,
271 (0x8000 << 16) | (0x9b7c >> 2),
272 0x00000000,
273 (0x8040 << 16) | (0x9b7c >> 2),
274 0x00000000,
275 (0x8000 << 16) | (0xe84 >> 2),
276 0x00000000,
277 (0x8040 << 16) | (0xe84 >> 2),
278 0x00000000,
279 (0x8000 << 16) | (0x89c0 >> 2),
280 0x00000000,
281 (0x8040 << 16) | (0x89c0 >> 2),
282 0x00000000,
283 (0x8000 << 16) | (0x914c >> 2),
284 0x00000000,
285 (0x8040 << 16) | (0x914c >> 2),
286 0x00000000,
287 (0x8000 << 16) | (0x8c20 >> 2),
288 0x00000000,
289 (0x8040 << 16) | (0x8c20 >> 2),
290 0x00000000,
291 (0x8000 << 16) | (0x9354 >> 2),
292 0x00000000,
293 (0x8040 << 16) | (0x9354 >> 2),
294 0x00000000,
295 (0x9c00 << 16) | (0x9060 >> 2),
296 0x00000000,
297 (0x9c00 << 16) | (0x9364 >> 2),
298 0x00000000,
299 (0x9c00 << 16) | (0x9100 >> 2),
300 0x00000000,
301 (0x9c00 << 16) | (0x913c >> 2),
302 0x00000000,
303 (0x8000 << 16) | (0x90e0 >> 2),
304 0x00000000,
305 (0x8000 << 16) | (0x90e4 >> 2),
306 0x00000000,
307 (0x8000 << 16) | (0x90e8 >> 2),
308 0x00000000,
309 (0x8040 << 16) | (0x90e0 >> 2),
310 0x00000000,
311 (0x8040 << 16) | (0x90e4 >> 2),
312 0x00000000,
313 (0x8040 << 16) | (0x90e8 >> 2),
314 0x00000000,
315 (0x9c00 << 16) | (0x8bcc >> 2),
316 0x00000000,
317 (0x9c00 << 16) | (0x8b24 >> 2),
318 0x00000000,
319 (0x9c00 << 16) | (0x88c4 >> 2),
320 0x00000000,
321 (0x9c00 << 16) | (0x8e50 >> 2),
322 0x00000000,
323 (0x9c00 << 16) | (0x8c0c >> 2),
324 0x00000000,
325 (0x9c00 << 16) | (0x8e58 >> 2),
326 0x00000000,
327 (0x9c00 << 16) | (0x8e5c >> 2),
328 0x00000000,
329 (0x9c00 << 16) | (0x9508 >> 2),
330 0x00000000,
331 (0x9c00 << 16) | (0x950c >> 2),
332 0x00000000,
333 (0x9c00 << 16) | (0x9494 >> 2),
334 0x00000000,
335 (0x9c00 << 16) | (0xac0c >> 2),
336 0x00000000,
337 (0x9c00 << 16) | (0xac10 >> 2),
338 0x00000000,
339 (0x9c00 << 16) | (0xac14 >> 2),
340 0x00000000,
341 (0x9c00 << 16) | (0xae00 >> 2),
342 0x00000000,
343 (0x9c00 << 16) | (0xac08 >> 2),
344 0x00000000,
345 (0x9c00 << 16) | (0x88d4 >> 2),
346 0x00000000,
347 (0x9c00 << 16) | (0x88c8 >> 2),
348 0x00000000,
349 (0x9c00 << 16) | (0x88cc >> 2),
350 0x00000000,
351 (0x9c00 << 16) | (0x89b0 >> 2),
352 0x00000000,
353 (0x9c00 << 16) | (0x8b10 >> 2),
354 0x00000000,
355 (0x9c00 << 16) | (0x8a14 >> 2),
356 0x00000000,
357 (0x9c00 << 16) | (0x9830 >> 2),
358 0x00000000,
359 (0x9c00 << 16) | (0x9834 >> 2),
360 0x00000000,
361 (0x9c00 << 16) | (0x9838 >> 2),
362 0x00000000,
363 (0x9c00 << 16) | (0x9a10 >> 2),
364 0x00000000,
365 (0x8000 << 16) | (0x9870 >> 2),
366 0x00000000,
367 (0x8000 << 16) | (0x9874 >> 2),
368 0x00000000,
369 (0x8001 << 16) | (0x9870 >> 2),
370 0x00000000,
371 (0x8001 << 16) | (0x9874 >> 2),
372 0x00000000,
373 (0x8040 << 16) | (0x9870 >> 2),
374 0x00000000,
375 (0x8040 << 16) | (0x9874 >> 2),
376 0x00000000,
377 (0x8041 << 16) | (0x9870 >> 2),
378 0x00000000,
379 (0x8041 << 16) | (0x9874 >> 2),
380 0x00000000,
381 0x00000000
382};
383
ce136e15 384static const u32 tahiti_golden_rlc_registers[] = {
205996c0
AD		 385 0xc424, 0xffffffff, 0x00601005,
386 0xc47c, 0xffffffff, 0x10104040,
387 0xc488, 0xffffffff, 0x0100000a,
388 0xc314, 0xffffffff, 0x00000800,
389 0xc30c, 0xffffffff, 0x800000f4,
390 0xf4a8, 0xffffffff, 0x00000000
391};
392
ce136e15 393static const u32 tahiti_golden_registers[] = {
205996c0
AD		 394 0x9a10, 0x00010000, 0x00018208,
395 0x9830, 0xffffffff, 0x00000000,
396 0x9834, 0xf00fffff, 0x00000400,
397 0x9838, 0x0002021c, 0x00020200,
398 0xc78, 0x00000080, 0x00000000,
399 0xd030, 0x000300c0, 0x00800040,
400 0xd830, 0x000300c0, 0x00800040,
401 0x5bb0, 0x000000f0, 0x00000070,
402 0x5bc0, 0x00200000, 0x50100000,
403 0x7030, 0x31000311, 0x00000011,
404 0x277c, 0x00000003, 0x000007ff,
405 0x240c, 0x000007ff, 0x00000000,
406 0x8a14, 0xf000001f, 0x00000007,
407 0x8b24, 0xffffffff, 0x00ffffff,
408 0x8b10, 0x0000ff0f, 0x00000000,
409 0x28a4c, 0x07ffffff, 0x4e000000,
410 0x28350, 0x3f3f3fff, 0x2a00126a,
411 0x30, 0x000000ff, 0x0040,
412 0x34, 0x00000040, 0x00004040,
413 0x9100, 0x07ffffff, 0x03000000,
414 0x8e88, 0x01ff1f3f, 0x00000000,
415 0x8e84, 0x01ff1f3f, 0x00000000,
416 0x9060, 0x0000007f, 0x00000020,
417 0x9508, 0x00010000, 0x00010000,
418 0xac14, 0x00000200, 0x000002fb,
419 0xac10, 0xffffffff, 0x0000543b,
420 0xac0c, 0xffffffff, 0xa9210876,
421 0x88d0, 0xffffffff, 0x000fff40,
422 0x88d4, 0x0000001f, 0x00000010,
423 0x1410, 0x20000000, 0x20fffed8,
424 0x15c0, 0x000c0fc0, 0x000c0400
425};
426
ce136e15 427static const u32 tahiti_golden_registers2[] = {
205996c0
AD		 428 0xc64, 0x00000001, 0x00000001
429};
430
ce136e15 431static const u32 pitcairn_golden_rlc_registers[] = {
205996c0
AD		 432 0xc424, 0xffffffff, 0x00601004,
433 0xc47c, 0xffffffff, 0x10102020,
434 0xc488, 0xffffffff, 0x01000020,
435 0xc314, 0xffffffff, 0x00000800,
436 0xc30c, 0xffffffff, 0x800000a4
437};
438
ce136e15 439static const u32 pitcairn_golden_registers[] = {
205996c0
AD		 440 0x9a10, 0x00010000, 0x00018208,
441 0x9830, 0xffffffff, 0x00000000,
442 0x9834, 0xf00fffff, 0x00000400,
443 0x9838, 0x0002021c, 0x00020200,
444 0xc78, 0x00000080, 0x00000000,
445 0xd030, 0x000300c0, 0x00800040,
446 0xd830, 0x000300c0, 0x00800040,
447 0x5bb0, 0x000000f0, 0x00000070,
448 0x5bc0, 0x00200000, 0x50100000,
449 0x7030, 0x31000311, 0x00000011,
450 0x2ae4, 0x00073ffe, 0x000022a2,
451 0x240c, 0x000007ff, 0x00000000,
452 0x8a14, 0xf000001f, 0x00000007,
453 0x8b24, 0xffffffff, 0x00ffffff,
454 0x8b10, 0x0000ff0f, 0x00000000,
455 0x28a4c, 0x07ffffff, 0x4e000000,
456 0x28350, 0x3f3f3fff, 0x2a00126a,
457 0x30, 0x000000ff, 0x0040,
458 0x34, 0x00000040, 0x00004040,
459 0x9100, 0x07ffffff, 0x03000000,
460 0x9060, 0x0000007f, 0x00000020,
461 0x9508, 0x00010000, 0x00010000,
462 0xac14, 0x000003ff, 0x000000f7,
463 0xac10, 0xffffffff, 0x00000000,
464 0xac0c, 0xffffffff, 0x32761054,
465 0x88d4, 0x0000001f, 0x00000010,
466 0x15c0, 0x000c0fc0, 0x000c0400
467};
468
ce136e15 469static const u32 verde_golden_rlc_registers[] = {
205996c0
AD		 470 0xc424, 0xffffffff, 0x033f1005,
471 0xc47c, 0xffffffff, 0x10808020,
472 0xc488, 0xffffffff, 0x00800008,
473 0xc314, 0xffffffff, 0x00001000,
474 0xc30c, 0xffffffff, 0x80010014
475};
476
ce136e15 477static const u32 verde_golden_registers[] = {
205996c0
AD		 478 0x9a10, 0x00010000, 0x00018208,
479 0x9830, 0xffffffff, 0x00000000,
480 0x9834, 0xf00fffff, 0x00000400,
481 0x9838, 0x0002021c, 0x00020200,
482 0xc78, 0x00000080, 0x00000000,
483 0xd030, 0x000300c0, 0x00800040,
484 0xd030, 0x000300c0, 0x00800040,
485 0xd830, 0x000300c0, 0x00800040,
486 0xd830, 0x000300c0, 0x00800040,
487 0x5bb0, 0x000000f0, 0x00000070,
488 0x5bc0, 0x00200000, 0x50100000,
489 0x7030, 0x31000311, 0x00000011,
490 0x2ae4, 0x00073ffe, 0x000022a2,
491 0x2ae4, 0x00073ffe, 0x000022a2,
492 0x2ae4, 0x00073ffe, 0x000022a2,
493 0x240c, 0x000007ff, 0x00000000,
494 0x240c, 0x000007ff, 0x00000000,
495 0x240c, 0x000007ff, 0x00000000,
496 0x8a14, 0xf000001f, 0x00000007,
497 0x8a14, 0xf000001f, 0x00000007,
498 0x8a14, 0xf000001f, 0x00000007,
499 0x8b24, 0xffffffff, 0x00ffffff,
500 0x8b10, 0x0000ff0f, 0x00000000,
501 0x28a4c, 0x07ffffff, 0x4e000000,
502 0x28350, 0x3f3f3fff, 0x0000124a,
503 0x28350, 0x3f3f3fff, 0x0000124a,
504 0x28350, 0x3f3f3fff, 0x0000124a,
505 0x30, 0x000000ff, 0x0040,
506 0x34, 0x00000040, 0x00004040,
507 0x9100, 0x07ffffff, 0x03000000,
508 0x9100, 0x07ffffff, 0x03000000,
509 0x8e88, 0x01ff1f3f, 0x00000000,
510 0x8e88, 0x01ff1f3f, 0x00000000,
511 0x8e88, 0x01ff1f3f, 0x00000000,
512 0x8e84, 0x01ff1f3f, 0x00000000,
513 0x8e84, 0x01ff1f3f, 0x00000000,
514 0x8e84, 0x01ff1f3f, 0x00000000,
515 0x9060, 0x0000007f, 0x00000020,
516 0x9508, 0x00010000, 0x00010000,
517 0xac14, 0x000003ff, 0x00000003,
518 0xac14, 0x000003ff, 0x00000003,
519 0xac14, 0x000003ff, 0x00000003,
520 0xac10, 0xffffffff, 0x00000000,
521 0xac10, 0xffffffff, 0x00000000,
522 0xac10, 0xffffffff, 0x00000000,
523 0xac0c, 0xffffffff, 0x00001032,
524 0xac0c, 0xffffffff, 0x00001032,
525 0xac0c, 0xffffffff, 0x00001032,
526 0x88d4, 0x0000001f, 0x00000010,
527 0x88d4, 0x0000001f, 0x00000010,
528 0x88d4, 0x0000001f, 0x00000010,
529 0x15c0, 0x000c0fc0, 0x000c0400
530};
531
ce136e15 532static const u32 oland_golden_rlc_registers[] = {
205996c0
AD		 533 0xc424, 0xffffffff, 0x00601005,
534 0xc47c, 0xffffffff, 0x10104040,
535 0xc488, 0xffffffff, 0x0100000a,
536 0xc314, 0xffffffff, 0x00000800,
537 0xc30c, 0xffffffff, 0x800000f4
538};
539
ce136e15 540static const u32 oland_golden_registers[] = {
205996c0
AD		 541 0x9a10, 0x00010000, 0x00018208,
542 0x9830, 0xffffffff, 0x00000000,
543 0x9834, 0xf00fffff, 0x00000400,
544 0x9838, 0x0002021c, 0x00020200,
545 0xc78, 0x00000080, 0x00000000,
546 0xd030, 0x000300c0, 0x00800040,
547 0xd830, 0x000300c0, 0x00800040,
548 0x5bb0, 0x000000f0, 0x00000070,
549 0x5bc0, 0x00200000, 0x50100000,
550 0x7030, 0x31000311, 0x00000011,
551 0x2ae4, 0x00073ffe, 0x000022a2,
552 0x240c, 0x000007ff, 0x00000000,
553 0x8a14, 0xf000001f, 0x00000007,
554 0x8b24, 0xffffffff, 0x00ffffff,
555 0x8b10, 0x0000ff0f, 0x00000000,
556 0x28a4c, 0x07ffffff, 0x4e000000,
557 0x28350, 0x3f3f3fff, 0x00000082,
558 0x30, 0x000000ff, 0x0040,
559 0x34, 0x00000040, 0x00004040,
560 0x9100, 0x07ffffff, 0x03000000,
561 0x9060, 0x0000007f, 0x00000020,
562 0x9508, 0x00010000, 0x00010000,
563 0xac14, 0x000003ff, 0x000000f3,
564 0xac10, 0xffffffff, 0x00000000,
565 0xac0c, 0xffffffff, 0x00003210,
566 0x88d4, 0x0000001f, 0x00000010,
567 0x15c0, 0x000c0fc0, 0x000c0400
568};
569
ce136e15 570static const u32 hainan_golden_registers[] = {
fffbdda4
AD		 571 0x9a10, 0x00010000, 0x00018208,
572 0x9830, 0xffffffff, 0x00000000,
573 0x9834, 0xf00fffff, 0x00000400,
574 0x9838, 0x0002021c, 0x00020200,
575 0xd0c0, 0xff000fff, 0x00000100,
576 0xd030, 0x000300c0, 0x00800040,
577 0xd8c0, 0xff000fff, 0x00000100,
578 0xd830, 0x000300c0, 0x00800040,
579 0x2ae4, 0x00073ffe, 0x000022a2,
580 0x240c, 0x000007ff, 0x00000000,
581 0x8a14, 0xf000001f, 0x00000007,
582 0x8b24, 0xffffffff, 0x00ffffff,
583 0x8b10, 0x0000ff0f, 0x00000000,
584 0x28a4c, 0x07ffffff, 0x4e000000,
585 0x28350, 0x3f3f3fff, 0x00000000,
586 0x30, 0x000000ff, 0x0040,
587 0x34, 0x00000040, 0x00004040,
588 0x9100, 0x03e00000, 0x03600000,
589 0x9060, 0x0000007f, 0x00000020,
590 0x9508, 0x00010000, 0x00010000,
591 0xac14, 0x000003ff, 0x000000f1,
592 0xac10, 0xffffffff, 0x00000000,
593 0xac0c, 0xffffffff, 0x00003210,
594 0x88d4, 0x0000001f, 0x00000010,
595 0x15c0, 0x000c0fc0, 0x000c0400
596};
597
ce136e15 598static const u32 hainan_golden_registers2[] = {
fffbdda4
AD		 599 0x98f8, 0xffffffff, 0x02010001
600};
601
ce136e15 602static const u32 tahiti_mgcg_cgcg_init[] = {
205996c0
AD		 603 0xc400, 0xffffffff, 0xfffffffc,
604 0x802c, 0xffffffff, 0xe0000000,
605 0x9a60, 0xffffffff, 0x00000100,
606 0x92a4, 0xffffffff, 0x00000100,
607 0xc164, 0xffffffff, 0x00000100,
608 0x9774, 0xffffffff, 0x00000100,
609 0x8984, 0xffffffff, 0x06000100,
610 0x8a18, 0xffffffff, 0x00000100,
611 0x92a0, 0xffffffff, 0x00000100,
612 0xc380, 0xffffffff, 0x00000100,
613 0x8b28, 0xffffffff, 0x00000100,
614 0x9144, 0xffffffff, 0x00000100,
615 0x8d88, 0xffffffff, 0x00000100,
616 0x8d8c, 0xffffffff, 0x00000100,
617 0x9030, 0xffffffff, 0x00000100,
618 0x9034, 0xffffffff, 0x00000100,
619 0x9038, 0xffffffff, 0x00000100,
620 0x903c, 0xffffffff, 0x00000100,
621 0xad80, 0xffffffff, 0x00000100,
622 0xac54, 0xffffffff, 0x00000100,
623 0x897c, 0xffffffff, 0x06000100,
624 0x9868, 0xffffffff, 0x00000100,
625 0x9510, 0xffffffff, 0x00000100,
626 0xaf04, 0xffffffff, 0x00000100,
627 0xae04, 0xffffffff, 0x00000100,
628 0x949c, 0xffffffff, 0x00000100,
629 0x802c, 0xffffffff, 0xe0000000,
630 0x9160, 0xffffffff, 0x00010000,
631 0x9164, 0xffffffff, 0x00030002,
632 0x9168, 0xffffffff, 0x00040007,
633 0x916c, 0xffffffff, 0x00060005,
634 0x9170, 0xffffffff, 0x00090008,
635 0x9174, 0xffffffff, 0x00020001,
636 0x9178, 0xffffffff, 0x00040003,
637 0x917c, 0xffffffff, 0x00000007,
638 0x9180, 0xffffffff, 0x00060005,
639 0x9184, 0xffffffff, 0x00090008,
640 0x9188, 0xffffffff, 0x00030002,
641 0x918c, 0xffffffff, 0x00050004,
642 0x9190, 0xffffffff, 0x00000008,
643 0x9194, 0xffffffff, 0x00070006,
644 0x9198, 0xffffffff, 0x000a0009,
645 0x919c, 0xffffffff, 0x00040003,
646 0x91a0, 0xffffffff, 0x00060005,
647 0x91a4, 0xffffffff, 0x00000009,
648 0x91a8, 0xffffffff, 0x00080007,
649 0x91ac, 0xffffffff, 0x000b000a,
650 0x91b0, 0xffffffff, 0x00050004,
651 0x91b4, 0xffffffff, 0x00070006,
652 0x91b8, 0xffffffff, 0x0008000b,
653 0x91bc, 0xffffffff, 0x000a0009,
654 0x91c0, 0xffffffff, 0x000d000c,
655 0x91c4, 0xffffffff, 0x00060005,
656 0x91c8, 0xffffffff, 0x00080007,
657 0x91cc, 0xffffffff, 0x0000000b,
658 0x91d0, 0xffffffff, 0x000a0009,
659 0x91d4, 0xffffffff, 0x000d000c,
660 0x91d8, 0xffffffff, 0x00070006,
661 0x91dc, 0xffffffff, 0x00090008,
662 0x91e0, 0xffffffff, 0x0000000c,
663 0x91e4, 0xffffffff, 0x000b000a,
664 0x91e8, 0xffffffff, 0x000e000d,
665 0x91ec, 0xffffffff, 0x00080007,
666 0x91f0, 0xffffffff, 0x000a0009,
667 0x91f4, 0xffffffff, 0x0000000d,
668 0x91f8, 0xffffffff, 0x000c000b,
669 0x91fc, 0xffffffff, 0x000f000e,
670 0x9200, 0xffffffff, 0x00090008,
671 0x9204, 0xffffffff, 0x000b000a,
672 0x9208, 0xffffffff, 0x000c000f,
673 0x920c, 0xffffffff, 0x000e000d,
674 0x9210, 0xffffffff, 0x00110010,
675 0x9214, 0xffffffff, 0x000a0009,
676 0x9218, 0xffffffff, 0x000c000b,
677 0x921c, 0xffffffff, 0x0000000f,
678 0x9220, 0xffffffff, 0x000e000d,
679 0x9224, 0xffffffff, 0x00110010,
680 0x9228, 0xffffffff, 0x000b000a,
681 0x922c, 0xffffffff, 0x000d000c,
682 0x9230, 0xffffffff, 0x00000010,
683 0x9234, 0xffffffff, 0x000f000e,
684 0x9238, 0xffffffff, 0x00120011,
685 0x923c, 0xffffffff, 0x000c000b,
686 0x9240, 0xffffffff, 0x000e000d,
687 0x9244, 0xffffffff, 0x00000011,
688 0x9248, 0xffffffff, 0x0010000f,
689 0x924c, 0xffffffff, 0x00130012,
690 0x9250, 0xffffffff, 0x000d000c,
691 0x9254, 0xffffffff, 0x000f000e,
692 0x9258, 0xffffffff, 0x00100013,
693 0x925c, 0xffffffff, 0x00120011,
694 0x9260, 0xffffffff, 0x00150014,
695 0x9264, 0xffffffff, 0x000e000d,
696 0x9268, 0xffffffff, 0x0010000f,
697 0x926c, 0xffffffff, 0x00000013,
698 0x9270, 0xffffffff, 0x00120011,
699 0x9274, 0xffffffff, 0x00150014,
700 0x9278, 0xffffffff, 0x000f000e,
701 0x927c, 0xffffffff, 0x00110010,
702 0x9280, 0xffffffff, 0x00000014,
703 0x9284, 0xffffffff, 0x00130012,
704 0x9288, 0xffffffff, 0x00160015,
705 0x928c, 0xffffffff, 0x0010000f,
706 0x9290, 0xffffffff, 0x00120011,
707 0x9294, 0xffffffff, 0x00000015,
708 0x9298, 0xffffffff, 0x00140013,
709 0x929c, 0xffffffff, 0x00170016,
710 0x9150, 0xffffffff, 0x96940200,
711 0x8708, 0xffffffff, 0x00900100,
712 0xc478, 0xffffffff, 0x00000080,
713 0xc404, 0xffffffff, 0x0020003f,
714 0x30, 0xffffffff, 0x0000001c,
715 0x34, 0x000f0000, 0x000f0000,
716 0x160c, 0xffffffff, 0x00000100,
717 0x1024, 0xffffffff, 0x00000100,
718 0x102c, 0x00000101, 0x00000000,
719 0x20a8, 0xffffffff, 0x00000104,
720 0x264c, 0x000c0000, 0x000c0000,
721 0x2648, 0x000c0000, 0x000c0000,
722 0x55e4, 0xff000fff, 0x00000100,
723 0x55e8, 0x00000001, 0x00000001,
724 0x2f50, 0x00000001, 0x00000001,
725 0x30cc, 0xc0000fff, 0x00000104,
726 0xc1e4, 0x00000001, 0x00000001,
727 0xd0c0, 0xfffffff0, 0x00000100,
728 0xd8c0, 0xfffffff0, 0x00000100
729};
730
ce136e15 731static const u32 pitcairn_mgcg_cgcg_init[] = {
205996c0
AD		 732 0xc400, 0xffffffff, 0xfffffffc,
733 0x802c, 0xffffffff, 0xe0000000,
734 0x9a60, 0xffffffff, 0x00000100,
735 0x92a4, 0xffffffff, 0x00000100,
736 0xc164, 0xffffffff, 0x00000100,
737 0x9774, 0xffffffff, 0x00000100,
738 0x8984, 0xffffffff, 0x06000100,
739 0x8a18, 0xffffffff, 0x00000100,
740 0x92a0, 0xffffffff, 0x00000100,
741 0xc380, 0xffffffff, 0x00000100,
742 0x8b28, 0xffffffff, 0x00000100,
743 0x9144, 0xffffffff, 0x00000100,
744 0x8d88, 0xffffffff, 0x00000100,
745 0x8d8c, 0xffffffff, 0x00000100,
746 0x9030, 0xffffffff, 0x00000100,
747 0x9034, 0xffffffff, 0x00000100,
748 0x9038, 0xffffffff, 0x00000100,
749 0x903c, 0xffffffff, 0x00000100,
750 0xad80, 0xffffffff, 0x00000100,
751 0xac54, 0xffffffff, 0x00000100,
752 0x897c, 0xffffffff, 0x06000100,
753 0x9868, 0xffffffff, 0x00000100,
754 0x9510, 0xffffffff, 0x00000100,
755 0xaf04, 0xffffffff, 0x00000100,
756 0xae04, 0xffffffff, 0x00000100,
757 0x949c, 0xffffffff, 0x00000100,
758 0x802c, 0xffffffff, 0xe0000000,
759 0x9160, 0xffffffff, 0x00010000,
760 0x9164, 0xffffffff, 0x00030002,
761 0x9168, 0xffffffff, 0x00040007,
762 0x916c, 0xffffffff, 0x00060005,
763 0x9170, 0xffffffff, 0x00090008,
764 0x9174, 0xffffffff, 0x00020001,
765 0x9178, 0xffffffff, 0x00040003,
766 0x917c, 0xffffffff, 0x00000007,
767 0x9180, 0xffffffff, 0x00060005,
768 0x9184, 0xffffffff, 0x00090008,
769 0x9188, 0xffffffff, 0x00030002,
770 0x918c, 0xffffffff, 0x00050004,
771 0x9190, 0xffffffff, 0x00000008,
772 0x9194, 0xffffffff, 0x00070006,
773 0x9198, 0xffffffff, 0x000a0009,
774 0x919c, 0xffffffff, 0x00040003,
775 0x91a0, 0xffffffff, 0x00060005,
776 0x91a4, 0xffffffff, 0x00000009,
777 0x91a8, 0xffffffff, 0x00080007,
778 0x91ac, 0xffffffff, 0x000b000a,
779 0x91b0, 0xffffffff, 0x00050004,
780 0x91b4, 0xffffffff, 0x00070006,
781 0x91b8, 0xffffffff, 0x0008000b,
782 0x91bc, 0xffffffff, 0x000a0009,
783 0x91c0, 0xffffffff, 0x000d000c,
784 0x9200, 0xffffffff, 0x00090008,
785 0x9204, 0xffffffff, 0x000b000a,
786 0x9208, 0xffffffff, 0x000c000f,
787 0x920c, 0xffffffff, 0x000e000d,
788 0x9210, 0xffffffff, 0x00110010,
789 0x9214, 0xffffffff, 0x000a0009,
790 0x9218, 0xffffffff, 0x000c000b,
791 0x921c, 0xffffffff, 0x0000000f,
792 0x9220, 0xffffffff, 0x000e000d,
793 0x9224, 0xffffffff, 0x00110010,
794 0x9228, 0xffffffff, 0x000b000a,
795 0x922c, 0xffffffff, 0x000d000c,
796 0x9230, 0xffffffff, 0x00000010,
797 0x9234, 0xffffffff, 0x000f000e,
798 0x9238, 0xffffffff, 0x00120011,
799 0x923c, 0xffffffff, 0x000c000b,
800 0x9240, 0xffffffff, 0x000e000d,
801 0x9244, 0xffffffff, 0x00000011,
802 0x9248, 0xffffffff, 0x0010000f,
803 0x924c, 0xffffffff, 0x00130012,
804 0x9250, 0xffffffff, 0x000d000c,
805 0x9254, 0xffffffff, 0x000f000e,
806 0x9258, 0xffffffff, 0x00100013,
807 0x925c, 0xffffffff, 0x00120011,
808 0x9260, 0xffffffff, 0x00150014,
809 0x9150, 0xffffffff, 0x96940200,
810 0x8708, 0xffffffff, 0x00900100,
811 0xc478, 0xffffffff, 0x00000080,
812 0xc404, 0xffffffff, 0x0020003f,
813 0x30, 0xffffffff, 0x0000001c,
814 0x34, 0x000f0000, 0x000f0000,
815 0x160c, 0xffffffff, 0x00000100,
816 0x1024, 0xffffffff, 0x00000100,
817 0x102c, 0x00000101, 0x00000000,
818 0x20a8, 0xffffffff, 0x00000104,
819 0x55e4, 0xff000fff, 0x00000100,
820 0x55e8, 0x00000001, 0x00000001,
821 0x2f50, 0x00000001, 0x00000001,
822 0x30cc, 0xc0000fff, 0x00000104,
823 0xc1e4, 0x00000001, 0x00000001,
824 0xd0c0, 0xfffffff0, 0x00000100,
825 0xd8c0, 0xfffffff0, 0x00000100
826};
827
ce136e15 828static const u32 verde_mgcg_cgcg_init[] = {
205996c0
AD		 829 0xc400, 0xffffffff, 0xfffffffc,
830 0x802c, 0xffffffff, 0xe0000000,
831 0x9a60, 0xffffffff, 0x00000100,
832 0x92a4, 0xffffffff, 0x00000100,
833 0xc164, 0xffffffff, 0x00000100,
834 0x9774, 0xffffffff, 0x00000100,
835 0x8984, 0xffffffff, 0x06000100,
836 0x8a18, 0xffffffff, 0x00000100,
837 0x92a0, 0xffffffff, 0x00000100,
838 0xc380, 0xffffffff, 0x00000100,
839 0x8b28, 0xffffffff, 0x00000100,
840 0x9144, 0xffffffff, 0x00000100,
841 0x8d88, 0xffffffff, 0x00000100,
842 0x8d8c, 0xffffffff, 0x00000100,
843 0x9030, 0xffffffff, 0x00000100,
844 0x9034, 0xffffffff, 0x00000100,
845 0x9038, 0xffffffff, 0x00000100,
846 0x903c, 0xffffffff, 0x00000100,
847 0xad80, 0xffffffff, 0x00000100,
848 0xac54, 0xffffffff, 0x00000100,
849 0x897c, 0xffffffff, 0x06000100,
850 0x9868, 0xffffffff, 0x00000100,
851 0x9510, 0xffffffff, 0x00000100,
852 0xaf04, 0xffffffff, 0x00000100,
853 0xae04, 0xffffffff, 0x00000100,
854 0x949c, 0xffffffff, 0x00000100,
855 0x802c, 0xffffffff, 0xe0000000,
856 0x9160, 0xffffffff, 0x00010000,
857 0x9164, 0xffffffff, 0x00030002,
858 0x9168, 0xffffffff, 0x00040007,
859 0x916c, 0xffffffff, 0x00060005,
860 0x9170, 0xffffffff, 0x00090008,
861 0x9174, 0xffffffff, 0x00020001,
862 0x9178, 0xffffffff, 0x00040003,
863 0x917c, 0xffffffff, 0x00000007,
864 0x9180, 0xffffffff, 0x00060005,
865 0x9184, 0xffffffff, 0x00090008,
866 0x9188, 0xffffffff, 0x00030002,
867 0x918c, 0xffffffff, 0x00050004,
868 0x9190, 0xffffffff, 0x00000008,
869 0x9194, 0xffffffff, 0x00070006,
870 0x9198, 0xffffffff, 0x000a0009,
871 0x919c, 0xffffffff, 0x00040003,
872 0x91a0, 0xffffffff, 0x00060005,
873 0x91a4, 0xffffffff, 0x00000009,
874 0x91a8, 0xffffffff, 0x00080007,
875 0x91ac, 0xffffffff, 0x000b000a,
876 0x91b0, 0xffffffff, 0x00050004,
877 0x91b4, 0xffffffff, 0x00070006,
878 0x91b8, 0xffffffff, 0x0008000b,
879 0x91bc, 0xffffffff, 0x000a0009,
880 0x91c0, 0xffffffff, 0x000d000c,
881 0x9200, 0xffffffff, 0x00090008,
882 0x9204, 0xffffffff, 0x000b000a,
883 0x9208, 0xffffffff, 0x000c000f,
884 0x920c, 0xffffffff, 0x000e000d,
885 0x9210, 0xffffffff, 0x00110010,
886 0x9214, 0xffffffff, 0x000a0009,
887 0x9218, 0xffffffff, 0x000c000b,
888 0x921c, 0xffffffff, 0x0000000f,
889 0x9220, 0xffffffff, 0x000e000d,
890 0x9224, 0xffffffff, 0x00110010,
891 0x9228, 0xffffffff, 0x000b000a,
892 0x922c, 0xffffffff, 0x000d000c,
893 0x9230, 0xffffffff, 0x00000010,
894 0x9234, 0xffffffff, 0x000f000e,
895 0x9238, 0xffffffff, 0x00120011,
896 0x923c, 0xffffffff, 0x000c000b,
897 0x9240, 0xffffffff, 0x000e000d,
898 0x9244, 0xffffffff, 0x00000011,
899 0x9248, 0xffffffff, 0x0010000f,
900 0x924c, 0xffffffff, 0x00130012,
901 0x9250, 0xffffffff, 0x000d000c,
902 0x9254, 0xffffffff, 0x000f000e,
903 0x9258, 0xffffffff, 0x00100013,
904 0x925c, 0xffffffff, 0x00120011,
905 0x9260, 0xffffffff, 0x00150014,
906 0x9150, 0xffffffff, 0x96940200,
907 0x8708, 0xffffffff, 0x00900100,
908 0xc478, 0xffffffff, 0x00000080,
909 0xc404, 0xffffffff, 0x0020003f,
910 0x30, 0xffffffff, 0x0000001c,
911 0x34, 0x000f0000, 0x000f0000,
912 0x160c, 0xffffffff, 0x00000100,
913 0x1024, 0xffffffff, 0x00000100,
914 0x102c, 0x00000101, 0x00000000,
915 0x20a8, 0xffffffff, 0x00000104,
916 0x264c, 0x000c0000, 0x000c0000,
917 0x2648, 0x000c0000, 0x000c0000,
918 0x55e4, 0xff000fff, 0x00000100,
919 0x55e8, 0x00000001, 0x00000001,
920 0x2f50, 0x00000001, 0x00000001,
921 0x30cc, 0xc0000fff, 0x00000104,
922 0xc1e4, 0x00000001, 0x00000001,
923 0xd0c0, 0xfffffff0, 0x00000100,
924 0xd8c0, 0xfffffff0, 0x00000100
925};
926
ce136e15 927static const u32 oland_mgcg_cgcg_init[] = {
205996c0
AD		 928 0xc400, 0xffffffff, 0xfffffffc,
929 0x802c, 0xffffffff, 0xe0000000,
930 0x9a60, 0xffffffff, 0x00000100,
931 0x92a4, 0xffffffff, 0x00000100,
932 0xc164, 0xffffffff, 0x00000100,
933 0x9774, 0xffffffff, 0x00000100,
934 0x8984, 0xffffffff, 0x06000100,
935 0x8a18, 0xffffffff, 0x00000100,
936 0x92a0, 0xffffffff, 0x00000100,
937 0xc380, 0xffffffff, 0x00000100,
938 0x8b28, 0xffffffff, 0x00000100,
939 0x9144, 0xffffffff, 0x00000100,
940 0x8d88, 0xffffffff, 0x00000100,
941 0x8d8c, 0xffffffff, 0x00000100,
942 0x9030, 0xffffffff, 0x00000100,
943 0x9034, 0xffffffff, 0x00000100,
944 0x9038, 0xffffffff, 0x00000100,
945 0x903c, 0xffffffff, 0x00000100,
946 0xad80, 0xffffffff, 0x00000100,
947 0xac54, 0xffffffff, 0x00000100,
948 0x897c, 0xffffffff, 0x06000100,
949 0x9868, 0xffffffff, 0x00000100,
950 0x9510, 0xffffffff, 0x00000100,
951 0xaf04, 0xffffffff, 0x00000100,
952 0xae04, 0xffffffff, 0x00000100,
953 0x949c, 0xffffffff, 0x00000100,
954 0x802c, 0xffffffff, 0xe0000000,
955 0x9160, 0xffffffff, 0x00010000,
956 0x9164, 0xffffffff, 0x00030002,
957 0x9168, 0xffffffff, 0x00040007,
958 0x916c, 0xffffffff, 0x00060005,
959 0x9170, 0xffffffff, 0x00090008,
960 0x9174, 0xffffffff, 0x00020001,
961 0x9178, 0xffffffff, 0x00040003,
962 0x917c, 0xffffffff, 0x00000007,
963 0x9180, 0xffffffff, 0x00060005,
964 0x9184, 0xffffffff, 0x00090008,
965 0x9188, 0xffffffff, 0x00030002,
966 0x918c, 0xffffffff, 0x00050004,
967 0x9190, 0xffffffff, 0x00000008,
968 0x9194, 0xffffffff, 0x00070006,
969 0x9198, 0xffffffff, 0x000a0009,
970 0x919c, 0xffffffff, 0x00040003,
971 0x91a0, 0xffffffff, 0x00060005,
972 0x91a4, 0xffffffff, 0x00000009,
973 0x91a8, 0xffffffff, 0x00080007,
974 0x91ac, 0xffffffff, 0x000b000a,
975 0x91b0, 0xffffffff, 0x00050004,
976 0x91b4, 0xffffffff, 0x00070006,
977 0x91b8, 0xffffffff, 0x0008000b,
978 0x91bc, 0xffffffff, 0x000a0009,
979 0x91c0, 0xffffffff, 0x000d000c,
980 0x91c4, 0xffffffff, 0x00060005,
981 0x91c8, 0xffffffff, 0x00080007,
982 0x91cc, 0xffffffff, 0x0000000b,
983 0x91d0, 0xffffffff, 0x000a0009,
984 0x91d4, 0xffffffff, 0x000d000c,
985 0x9150, 0xffffffff, 0x96940200,
986 0x8708, 0xffffffff, 0x00900100,
987 0xc478, 0xffffffff, 0x00000080,
988 0xc404, 0xffffffff, 0x0020003f,
989 0x30, 0xffffffff, 0x0000001c,
990 0x34, 0x000f0000, 0x000f0000,
991 0x160c, 0xffffffff, 0x00000100,
992 0x1024, 0xffffffff, 0x00000100,
993 0x102c, 0x00000101, 0x00000000,
994 0x20a8, 0xffffffff, 0x00000104,
995 0x264c, 0x000c0000, 0x000c0000,
996 0x2648, 0x000c0000, 0x000c0000,
997 0x55e4, 0xff000fff, 0x00000100,
998 0x55e8, 0x00000001, 0x00000001,
999 0x2f50, 0x00000001, 0x00000001,
1000 0x30cc, 0xc0000fff, 0x00000104,
1001 0xc1e4, 0x00000001, 0x00000001,
1002 0xd0c0, 0xfffffff0, 0x00000100,
1003 0xd8c0, 0xfffffff0, 0x00000100
1004};
1005
ce136e15 1006static const u32 hainan_mgcg_cgcg_init[] = {
fffbdda4
AD		 1007 0xc400, 0xffffffff, 0xfffffffc,
1008 0x802c, 0xffffffff, 0xe0000000,
1009 0x9a60, 0xffffffff, 0x00000100,
1010 0x92a4, 0xffffffff, 0x00000100,
1011 0xc164, 0xffffffff, 0x00000100,
1012 0x9774, 0xffffffff, 0x00000100,
1013 0x8984, 0xffffffff, 0x06000100,
1014 0x8a18, 0xffffffff, 0x00000100,
1015 0x92a0, 0xffffffff, 0x00000100,
1016 0xc380, 0xffffffff, 0x00000100,
1017 0x8b28, 0xffffffff, 0x00000100,
1018 0x9144, 0xffffffff, 0x00000100,
1019 0x8d88, 0xffffffff, 0x00000100,
1020 0x8d8c, 0xffffffff, 0x00000100,
1021 0x9030, 0xffffffff, 0x00000100,
1022 0x9034, 0xffffffff, 0x00000100,
1023 0x9038, 0xffffffff, 0x00000100,
1024 0x903c, 0xffffffff, 0x00000100,
1025 0xad80, 0xffffffff, 0x00000100,
1026 0xac54, 0xffffffff, 0x00000100,
1027 0x897c, 0xffffffff, 0x06000100,
1028 0x9868, 0xffffffff, 0x00000100,
1029 0x9510, 0xffffffff, 0x00000100,
1030 0xaf04, 0xffffffff, 0x00000100,
1031 0xae04, 0xffffffff, 0x00000100,
1032 0x949c, 0xffffffff, 0x00000100,
1033 0x802c, 0xffffffff, 0xe0000000,
1034 0x9160, 0xffffffff, 0x00010000,
1035 0x9164, 0xffffffff, 0x00030002,
1036 0x9168, 0xffffffff, 0x00040007,
1037 0x916c, 0xffffffff, 0x00060005,
1038 0x9170, 0xffffffff, 0x00090008,
1039 0x9174, 0xffffffff, 0x00020001,
1040 0x9178, 0xffffffff, 0x00040003,
1041 0x917c, 0xffffffff, 0x00000007,
1042 0x9180, 0xffffffff, 0x00060005,
1043 0x9184, 0xffffffff, 0x00090008,
1044 0x9188, 0xffffffff, 0x00030002,
1045 0x918c, 0xffffffff, 0x00050004,
1046 0x9190, 0xffffffff, 0x00000008,
1047 0x9194, 0xffffffff, 0x00070006,
1048 0x9198, 0xffffffff, 0x000a0009,
1049 0x919c, 0xffffffff, 0x00040003,
1050 0x91a0, 0xffffffff, 0x00060005,
1051 0x91a4, 0xffffffff, 0x00000009,
1052 0x91a8, 0xffffffff, 0x00080007,
1053 0x91ac, 0xffffffff, 0x000b000a,
1054 0x91b0, 0xffffffff, 0x00050004,
1055 0x91b4, 0xffffffff, 0x00070006,
1056 0x91b8, 0xffffffff, 0x0008000b,
1057 0x91bc, 0xffffffff, 0x000a0009,
1058 0x91c0, 0xffffffff, 0x000d000c,
1059 0x91c4, 0xffffffff, 0x00060005,
1060 0x91c8, 0xffffffff, 0x00080007,
1061 0x91cc, 0xffffffff, 0x0000000b,
1062 0x91d0, 0xffffffff, 0x000a0009,
1063 0x91d4, 0xffffffff, 0x000d000c,
1064 0x9150, 0xffffffff, 0x96940200,
1065 0x8708, 0xffffffff, 0x00900100,
1066 0xc478, 0xffffffff, 0x00000080,
1067 0xc404, 0xffffffff, 0x0020003f,
1068 0x30, 0xffffffff, 0x0000001c,
1069 0x34, 0x000f0000, 0x000f0000,
1070 0x160c, 0xffffffff, 0x00000100,
1071 0x1024, 0xffffffff, 0x00000100,
1072 0x20a8, 0xffffffff, 0x00000104,
1073 0x264c, 0x000c0000, 0x000c0000,
1074 0x2648, 0x000c0000, 0x000c0000,
1075 0x2f50, 0x00000001, 0x00000001,
1076 0x30cc, 0xc0000fff, 0x00000104,
1077 0xc1e4, 0x00000001, 0x00000001,
1078 0xd0c0, 0xfffffff0, 0x00000100,
1079 0xd8c0, 0xfffffff0, 0x00000100
1080};
1081
ce136e15 1082static u32 verde_pg_init[] = {
205996c0
AD		 1083 0x353c, 0xffffffff, 0x40000,
1084 0x3538, 0xffffffff, 0x200010ff,
1085 0x353c, 0xffffffff, 0x0,
1086 0x353c, 0xffffffff, 0x0,
1087 0x353c, 0xffffffff, 0x0,
1088 0x353c, 0xffffffff, 0x0,
1089 0x353c, 0xffffffff, 0x0,
1090 0x353c, 0xffffffff, 0x7007,
1091 0x3538, 0xffffffff, 0x300010ff,
1092 0x353c, 0xffffffff, 0x0,
1093 0x353c, 0xffffffff, 0x0,
1094 0x353c, 0xffffffff, 0x0,
1095 0x353c, 0xffffffff, 0x0,
1096 0x353c, 0xffffffff, 0x0,
1097 0x353c, 0xffffffff, 0x400000,
1098 0x3538, 0xffffffff, 0x100010ff,
1099 0x353c, 0xffffffff, 0x0,
1100 0x353c, 0xffffffff, 0x0,
1101 0x353c, 0xffffffff, 0x0,
1102 0x353c, 0xffffffff, 0x0,
1103 0x353c, 0xffffffff, 0x0,
1104 0x353c, 0xffffffff, 0x120200,
1105 0x3538, 0xffffffff, 0x500010ff,
1106 0x353c, 0xffffffff, 0x0,
1107 0x353c, 0xffffffff, 0x0,
1108 0x353c, 0xffffffff, 0x0,
1109 0x353c, 0xffffffff, 0x0,
1110 0x353c, 0xffffffff, 0x0,
1111 0x353c, 0xffffffff, 0x1e1e16,
1112 0x3538, 0xffffffff, 0x600010ff,
1113 0x353c, 0xffffffff, 0x0,
1114 0x353c, 0xffffffff, 0x0,
1115 0x353c, 0xffffffff, 0x0,
1116 0x353c, 0xffffffff, 0x0,
1117 0x353c, 0xffffffff, 0x0,
1118 0x353c, 0xffffffff, 0x171f1e,
1119 0x3538, 0xffffffff, 0x700010ff,
1120 0x353c, 0xffffffff, 0x0,
1121 0x353c, 0xffffffff, 0x0,
1122 0x353c, 0xffffffff, 0x0,
1123 0x353c, 0xffffffff, 0x0,
1124 0x353c, 0xffffffff, 0x0,
1125 0x353c, 0xffffffff, 0x0,
1126 0x3538, 0xffffffff, 0x9ff,
1127 0x3500, 0xffffffff, 0x0,
1128 0x3504, 0xffffffff, 0x10000800,
1129 0x3504, 0xffffffff, 0xf,
1130 0x3504, 0xffffffff, 0xf,
1131 0x3500, 0xffffffff, 0x4,
1132 0x3504, 0xffffffff, 0x1000051e,
1133 0x3504, 0xffffffff, 0xffff,
1134 0x3504, 0xffffffff, 0xffff,
1135 0x3500, 0xffffffff, 0x8,
1136 0x3504, 0xffffffff, 0x80500,
1137 0x3500, 0xffffffff, 0x12,
1138 0x3504, 0xffffffff, 0x9050c,
1139 0x3500, 0xffffffff, 0x1d,
1140 0x3504, 0xffffffff, 0xb052c,
1141 0x3500, 0xffffffff, 0x2a,
1142 0x3504, 0xffffffff, 0x1053e,
1143 0x3500, 0xffffffff, 0x2d,
1144 0x3504, 0xffffffff, 0x10546,
1145 0x3500, 0xffffffff, 0x30,
1146 0x3504, 0xffffffff, 0xa054e,
1147 0x3500, 0xffffffff, 0x3c,
1148 0x3504, 0xffffffff, 0x1055f,
1149 0x3500, 0xffffffff, 0x3f,
1150 0x3504, 0xffffffff, 0x10567,
1151 0x3500, 0xffffffff, 0x42,
1152 0x3504, 0xffffffff, 0x1056f,
1153 0x3500, 0xffffffff, 0x45,
1154 0x3504, 0xffffffff, 0x10572,
1155 0x3500, 0xffffffff, 0x48,
1156 0x3504, 0xffffffff, 0x20575,
1157 0x3500, 0xffffffff, 0x4c,
1158 0x3504, 0xffffffff, 0x190801,
1159 0x3500, 0xffffffff, 0x67,
1160 0x3504, 0xffffffff, 0x1082a,
1161 0x3500, 0xffffffff, 0x6a,
1162 0x3504, 0xffffffff, 0x1b082d,
1163 0x3500, 0xffffffff, 0x87,
1164 0x3504, 0xffffffff, 0x310851,
1165 0x3500, 0xffffffff, 0xba,
1166 0x3504, 0xffffffff, 0x891,
1167 0x3500, 0xffffffff, 0xbc,
1168 0x3504, 0xffffffff, 0x893,
1169 0x3500, 0xffffffff, 0xbe,
1170 0x3504, 0xffffffff, 0x20895,
1171 0x3500, 0xffffffff, 0xc2,
1172 0x3504, 0xffffffff, 0x20899,
1173 0x3500, 0xffffffff, 0xc6,
1174 0x3504, 0xffffffff, 0x2089d,
1175 0x3500, 0xffffffff, 0xca,
1176 0x3504, 0xffffffff, 0x8a1,
1177 0x3500, 0xffffffff, 0xcc,
1178 0x3504, 0xffffffff, 0x8a3,
1179 0x3500, 0xffffffff, 0xce,
1180 0x3504, 0xffffffff, 0x308a5,
1181 0x3500, 0xffffffff, 0xd3,
1182 0x3504, 0xffffffff, 0x6d08cd,
1183 0x3500, 0xffffffff, 0x142,
1184 0x3504, 0xffffffff, 0x2000095a,
1185 0x3504, 0xffffffff, 0x1,
1186 0x3500, 0xffffffff, 0x144,
1187 0x3504, 0xffffffff, 0x301f095b,
1188 0x3500, 0xffffffff, 0x165,
1189 0x3504, 0xffffffff, 0xc094d,
1190 0x3500, 0xffffffff, 0x173,
1191 0x3504, 0xffffffff, 0xf096d,
1192 0x3500, 0xffffffff, 0x184,
1193 0x3504, 0xffffffff, 0x15097f,
1194 0x3500, 0xffffffff, 0x19b,
1195 0x3504, 0xffffffff, 0xc0998,
1196 0x3500, 0xffffffff, 0x1a9,
1197 0x3504, 0xffffffff, 0x409a7,
1198 0x3500, 0xffffffff, 0x1af,
1199 0x3504, 0xffffffff, 0xcdc,
1200 0x3500, 0xffffffff, 0x1b1,
1201 0x3504, 0xffffffff, 0x800,
1202 0x3508, 0xffffffff, 0x6c9b2000,
1203 0x3510, 0xfc00, 0x2000,
1204 0x3544, 0xffffffff, 0xfc0,
1205 0x28d4, 0x00000100, 0x100
1206};
1207
1208static void si_init_golden_registers(struct radeon_device *rdev)
1209{
1210 switch (rdev->family) {
1211 case CHIP_TAHITI:
1212 radeon_program_register_sequence(rdev,
1213 tahiti_golden_registers,
1214 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1215 radeon_program_register_sequence(rdev,
1216 tahiti_golden_rlc_registers,
1217 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1218 radeon_program_register_sequence(rdev,
1219 tahiti_mgcg_cgcg_init,
1220 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1221 radeon_program_register_sequence(rdev,
1222 tahiti_golden_registers2,
1223 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1224 break;
1225 case CHIP_PITCAIRN:
1226 radeon_program_register_sequence(rdev,
1227 pitcairn_golden_registers,
1228 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1229 radeon_program_register_sequence(rdev,
1230 pitcairn_golden_rlc_registers,
1231 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1232 radeon_program_register_sequence(rdev,
1233 pitcairn_mgcg_cgcg_init,
1234 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1235 break;
1236 case CHIP_VERDE:
1237 radeon_program_register_sequence(rdev,
1238 verde_golden_registers,
1239 (const u32)ARRAY_SIZE(verde_golden_registers));
1240 radeon_program_register_sequence(rdev,
1241 verde_golden_rlc_registers,
1242 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1243 radeon_program_register_sequence(rdev,
1244 verde_mgcg_cgcg_init,
1245 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1246 radeon_program_register_sequence(rdev,
1247 verde_pg_init,
1248 (const u32)ARRAY_SIZE(verde_pg_init));
1249 break;
1250 case CHIP_OLAND:
1251 radeon_program_register_sequence(rdev,
1252 oland_golden_registers,
1253 (const u32)ARRAY_SIZE(oland_golden_registers));
1254 radeon_program_register_sequence(rdev,
1255 oland_golden_rlc_registers,
1256 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1257 radeon_program_register_sequence(rdev,
1258 oland_mgcg_cgcg_init,
1259 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1260 break;
fffbdda4
AD		 1261 case CHIP_HAINAN:
1262 radeon_program_register_sequence(rdev,
1263 hainan_golden_registers,
1264 (const u32)ARRAY_SIZE(hainan_golden_registers));
1265 radeon_program_register_sequence(rdev,
1266 hainan_golden_registers2,
1267 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1268 radeon_program_register_sequence(rdev,
1269 hainan_mgcg_cgcg_init,
1270 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1271 break;
205996c0
AD		 1272 default:
1273 break;
1274 }
1275}
1276
4af692f6
AD		 1277/**
1278 * si_get_allowed_info_register - fetch the register for the info ioctl
1279 *
1280 * @rdev: radeon_device pointer
1281 * @reg: register offset in bytes
1282 * @val: register value
1283 *
1284 * Returns 0 for success or -EINVAL for an invalid register
1285 *
1286 */
1287int si_get_allowed_info_register(struct radeon_device *rdev,
1288 u32 reg, u32 *val)
1289{
1290 switch (reg) {
1291 case GRBM_STATUS:
1292 case GRBM_STATUS2:
1293 case GRBM_STATUS_SE0:
1294 case GRBM_STATUS_SE1:
1295 case SRBM_STATUS:
1296 case SRBM_STATUS2:
1297 case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1298 case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1299 case UVD_STATUS:
1300 *val = RREG32(reg);
1301 return 0;
1302 default:
1303 return -EINVAL;
1304 }
1305}
1306
454d2e2a
AD		 1307#define PCIE_BUS_CLK 10000
1308#define TCLK (PCIE_BUS_CLK / 10)
1309
1310/**
1311 * si_get_xclk - get the xclk
1312 *
1313 * @rdev: radeon_device pointer
1314 *
1315 * Returns the reference clock used by the gfx engine
1316 * (SI).
1317 */
1318u32 si_get_xclk(struct radeon_device *rdev)
1319{
3cf8bb1a 1320 u32 reference_clock = rdev->clock.spll.reference_freq;
454d2e2a
AD		 1321 u32 tmp;
1322
1323 tmp = RREG32(CG_CLKPIN_CNTL_2);
1324 if (tmp & MUX_TCLK_TO_XCLK)
1325 return TCLK;
1326
1327 tmp = RREG32(CG_CLKPIN_CNTL);
1328 if (tmp & XTALIN_DIVIDE)
1329 return reference_clock / 4;
1330
1331 return reference_clock;
1332}
1333
1bd47d2e
AD		 1334/* get temperature in millidegrees */
1335int si_get_temp(struct radeon_device *rdev)
1336{
1337 u32 temp;
1338 int actual_temp = 0;
1339
1340 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1341 CTF_TEMP_SHIFT;
1342
1343 if (temp & 0x200)
1344 actual_temp = 255;
1345 else
1346 actual_temp = temp & 0x1ff;
1347
1348 actual_temp = (actual_temp * 1000);
1349
1350 return actual_temp;
1351}
1352
8b074dd6
AD		 1353#define TAHITI_IO_MC_REGS_SIZE 36
1354
1355static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1356 {0x0000006f, 0x03044000},
1357 {0x00000070, 0x0480c018},
1358 {0x00000071, 0x00000040},
1359 {0x00000072, 0x01000000},
1360 {0x00000074, 0x000000ff},
1361 {0x00000075, 0x00143400},
1362 {0x00000076, 0x08ec0800},
1363 {0x00000077, 0x040000cc},
1364 {0x00000079, 0x00000000},
1365 {0x0000007a, 0x21000409},
1366 {0x0000007c, 0x00000000},
1367 {0x0000007d, 0xe8000000},
1368 {0x0000007e, 0x044408a8},
1369 {0x0000007f, 0x00000003},
1370 {0x00000080, 0x00000000},
1371 {0x00000081, 0x01000000},
1372 {0x00000082, 0x02000000},
1373 {0x00000083, 0x00000000},
1374 {0x00000084, 0xe3f3e4f4},
1375 {0x00000085, 0x00052024},
1376 {0x00000087, 0x00000000},
1377 {0x00000088, 0x66036603},
1378 {0x00000089, 0x01000000},
1379 {0x0000008b, 0x1c0a0000},
1380 {0x0000008c, 0xff010000},
1381 {0x0000008e, 0xffffefff},
1382 {0x0000008f, 0xfff3efff},
1383 {0x00000090, 0xfff3efbf},
1384 {0x00000094, 0x00101101},
1385 {0x00000095, 0x00000fff},
1386 {0x00000096, 0x00116fff},
1387 {0x00000097, 0x60010000},
1388 {0x00000098, 0x10010000},
1389 {0x00000099, 0x00006000},
1390 {0x0000009a, 0x00001000},
1391 {0x0000009f, 0x00a77400}
1392};
1393
1394static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1395 {0x0000006f, 0x03044000},
1396 {0x00000070, 0x0480c018},
1397 {0x00000071, 0x00000040},
1398 {0x00000072, 0x01000000},
1399 {0x00000074, 0x000000ff},
1400 {0x00000075, 0x00143400},
1401 {0x00000076, 0x08ec0800},
1402 {0x00000077, 0x040000cc},
1403 {0x00000079, 0x00000000},
1404 {0x0000007a, 0x21000409},
1405 {0x0000007c, 0x00000000},
1406 {0x0000007d, 0xe8000000},
1407 {0x0000007e, 0x044408a8},
1408 {0x0000007f, 0x00000003},
1409 {0x00000080, 0x00000000},
1410 {0x00000081, 0x01000000},
1411 {0x00000082, 0x02000000},
1412 {0x00000083, 0x00000000},
1413 {0x00000084, 0xe3f3e4f4},
1414 {0x00000085, 0x00052024},
1415 {0x00000087, 0x00000000},
1416 {0x00000088, 0x66036603},
1417 {0x00000089, 0x01000000},
1418 {0x0000008b, 0x1c0a0000},
1419 {0x0000008c, 0xff010000},
1420 {0x0000008e, 0xffffefff},
1421 {0x0000008f, 0xfff3efff},
1422 {0x00000090, 0xfff3efbf},
1423 {0x00000094, 0x00101101},
1424 {0x00000095, 0x00000fff},
1425 {0x00000096, 0x00116fff},
1426 {0x00000097, 0x60010000},
1427 {0x00000098, 0x10010000},
1428 {0x00000099, 0x00006000},
1429 {0x0000009a, 0x00001000},
1430 {0x0000009f, 0x00a47400}
1431};
1432
1433static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1434 {0x0000006f, 0x03044000},
1435 {0x00000070, 0x0480c018},
1436 {0x00000071, 0x00000040},
1437 {0x00000072, 0x01000000},
1438 {0x00000074, 0x000000ff},
1439 {0x00000075, 0x00143400},
1440 {0x00000076, 0x08ec0800},
1441 {0x00000077, 0x040000cc},
1442 {0x00000079, 0x00000000},
1443 {0x0000007a, 0x21000409},
1444 {0x0000007c, 0x00000000},
1445 {0x0000007d, 0xe8000000},
1446 {0x0000007e, 0x044408a8},
1447 {0x0000007f, 0x00000003},
1448 {0x00000080, 0x00000000},
1449 {0x00000081, 0x01000000},
1450 {0x00000082, 0x02000000},
1451 {0x00000083, 0x00000000},
1452 {0x00000084, 0xe3f3e4f4},
1453 {0x00000085, 0x00052024},
1454 {0x00000087, 0x00000000},
1455 {0x00000088, 0x66036603},
1456 {0x00000089, 0x01000000},
1457 {0x0000008b, 0x1c0a0000},
1458 {0x0000008c, 0xff010000},
1459 {0x0000008e, 0xffffefff},
1460 {0x0000008f, 0xfff3efff},
1461 {0x00000090, 0xfff3efbf},
1462 {0x00000094, 0x00101101},
1463 {0x00000095, 0x00000fff},
1464 {0x00000096, 0x00116fff},
1465 {0x00000097, 0x60010000},
1466 {0x00000098, 0x10010000},
1467 {0x00000099, 0x00006000},
1468 {0x0000009a, 0x00001000},
1469 {0x0000009f, 0x00a37400}
1470};
1471
bcc7f5d2
AD		 1472static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1473 {0x0000006f, 0x03044000},
1474 {0x00000070, 0x0480c018},
1475 {0x00000071, 0x00000040},
1476 {0x00000072, 0x01000000},
1477 {0x00000074, 0x000000ff},
1478 {0x00000075, 0x00143400},
1479 {0x00000076, 0x08ec0800},
1480 {0x00000077, 0x040000cc},
1481 {0x00000079, 0x00000000},
1482 {0x0000007a, 0x21000409},
1483 {0x0000007c, 0x00000000},
1484 {0x0000007d, 0xe8000000},
1485 {0x0000007e, 0x044408a8},
1486 {0x0000007f, 0x00000003},
1487 {0x00000080, 0x00000000},
1488 {0x00000081, 0x01000000},
1489 {0x00000082, 0x02000000},
1490 {0x00000083, 0x00000000},
1491 {0x00000084, 0xe3f3e4f4},
1492 {0x00000085, 0x00052024},
1493 {0x00000087, 0x00000000},
1494 {0x00000088, 0x66036603},
1495 {0x00000089, 0x01000000},
1496 {0x0000008b, 0x1c0a0000},
1497 {0x0000008c, 0xff010000},
1498 {0x0000008e, 0xffffefff},
1499 {0x0000008f, 0xfff3efff},
1500 {0x00000090, 0xfff3efbf},
1501 {0x00000094, 0x00101101},
1502 {0x00000095, 0x00000fff},
1503 {0x00000096, 0x00116fff},
1504 {0x00000097, 0x60010000},
1505 {0x00000098, 0x10010000},
1506 {0x00000099, 0x00006000},
1507 {0x0000009a, 0x00001000},
1508 {0x0000009f, 0x00a17730}
1509};
1510
c04c00b4
AD		 1511static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1512 {0x0000006f, 0x03044000},
1513 {0x00000070, 0x0480c018},
1514 {0x00000071, 0x00000040},
1515 {0x00000072, 0x01000000},
1516 {0x00000074, 0x000000ff},
1517 {0x00000075, 0x00143400},
1518 {0x00000076, 0x08ec0800},
1519 {0x00000077, 0x040000cc},
1520 {0x00000079, 0x00000000},
1521 {0x0000007a, 0x21000409},
1522 {0x0000007c, 0x00000000},
1523 {0x0000007d, 0xe8000000},
1524 {0x0000007e, 0x044408a8},
1525 {0x0000007f, 0x00000003},
1526 {0x00000080, 0x00000000},
1527 {0x00000081, 0x01000000},
1528 {0x00000082, 0x02000000},
1529 {0x00000083, 0x00000000},
1530 {0x00000084, 0xe3f3e4f4},
1531 {0x00000085, 0x00052024},
1532 {0x00000087, 0x00000000},
1533 {0x00000088, 0x66036603},
1534 {0x00000089, 0x01000000},
1535 {0x0000008b, 0x1c0a0000},
1536 {0x0000008c, 0xff010000},
1537 {0x0000008e, 0xffffefff},
1538 {0x0000008f, 0xfff3efff},
1539 {0x00000090, 0xfff3efbf},
1540 {0x00000094, 0x00101101},
1541 {0x00000095, 0x00000fff},
1542 {0x00000096, 0x00116fff},
1543 {0x00000097, 0x60010000},
1544 {0x00000098, 0x10010000},
1545 {0x00000099, 0x00006000},
1546 {0x0000009a, 0x00001000},
1547 {0x0000009f, 0x00a07730}
1548};
1549
8b074dd6 1550/* ucode loading */
6c7bccea 1551int si_mc_load_microcode(struct radeon_device *rdev)
8b074dd6 1552{
629bd33c
AD		 1553 const __be32 *fw_data = NULL;
1554 const __le32 *new_fw_data = NULL;
ddbbd3be 1555 u32 running;
629bd33c
AD		 1556 u32 *io_mc_regs = NULL;
1557 const __le32 *new_io_mc_regs = NULL;
8c79bae6 1558 int i, regs_size, ucode_size;
8b074dd6
AD		 1559
1560 if (!rdev->mc_fw)
1561 return -EINVAL;
1562
629bd33c
AD		 1563 if (rdev->new_fw) {
1564 const struct mc_firmware_header_v1_0 *hdr =
1565 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1566
1567 radeon_ucode_print_mc_hdr(&hdr->header);
1568 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1569 new_io_mc_regs = (const __le32 *)
1570 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1571 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1572 new_fw_data = (const __le32 *)
1573 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1574 } else {
1575 ucode_size = rdev->mc_fw->size / 4;
8c79bae6 1576
629bd33c
AD		 1577 switch (rdev->family) {
1578 case CHIP_TAHITI:
1579 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1580 regs_size = TAHITI_IO_MC_REGS_SIZE;
1581 break;
1582 case CHIP_PITCAIRN:
1583 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1584 regs_size = TAHITI_IO_MC_REGS_SIZE;
1585 break;
1586 case CHIP_VERDE:
1587 default:
1588 io_mc_regs = (u32 *)&verde_io_mc_regs;
1589 regs_size = TAHITI_IO_MC_REGS_SIZE;
1590 break;
1591 case CHIP_OLAND:
1592 io_mc_regs = (u32 *)&oland_io_mc_regs;
1593 regs_size = TAHITI_IO_MC_REGS_SIZE;
1594 break;
1595 case CHIP_HAINAN:
1596 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1597 regs_size = TAHITI_IO_MC_REGS_SIZE;
1598 break;
1599 }
1600 fw_data = (const __be32 *)rdev->mc_fw->data;
8b074dd6
AD		 1601 }
1602
1603 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1604
1605 if (running == 0) {
8b074dd6
AD		 1606 /* reset the engine and set to writable */
1607 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1608 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1609
1610 /* load mc io regs */
1611 for (i = 0; i < regs_size; i++) {
629bd33c
AD		 1612 if (rdev->new_fw) {
1613 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1614 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1615 } else {
1616 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1617 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1618 }
8b074dd6
AD		 1619 }
1620 /* load the MC ucode */
629bd33c
AD		 1621 for (i = 0; i < ucode_size; i++) {
1622 if (rdev->new_fw)
1623 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1624 else
1625 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1626 }
8b074dd6
AD		 1627
1628 /* put the engine back into the active state */
1629 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1630 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1631 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1632
1633 /* wait for training to complete */
1634 for (i = 0; i < rdev->usec_timeout; i++) {
1635 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1636 break;
1637 udelay(1);
1638 }
1639 for (i = 0; i < rdev->usec_timeout; i++) {
1640 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1641 break;
1642 udelay(1);
1643 }
8b074dd6
AD		 1644 }
1645
1646 return 0;
1647}
1648
0f0de06c
AD		 1649static int si_init_microcode(struct radeon_device *rdev)
1650{
0f0de06c 1651 const char *chip_name;
629bd33c 1652 const char *new_chip_name;
0f0de06c 1653 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1ebe9280 1654 size_t smc_req_size, mc2_req_size;
0f0de06c
AD		 1655 char fw_name[30];
1656 int err;
629bd33c 1657 int new_fw = 0;
861c7fde 1658 bool new_smc = false;
ef736d39 1659 bool si58_fw = false;
4e6e98b1 1660 bool banks2_fw = false;
0f0de06c
AD		 1661
1662 DRM_DEBUG("\n");
1663
0f0de06c
AD		 1664 switch (rdev->family) {
1665 case CHIP_TAHITI:
1666 chip_name = "TAHITI";
629bd33c 1667 new_chip_name = "tahiti";
0f0de06c
AD		 1668 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1669 me_req_size = SI_PM4_UCODE_SIZE * 4;
1670 ce_req_size = SI_CE_UCODE_SIZE * 4;
1671 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1672 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1673 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
a9e61410 1674 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1675 break;
1676 case CHIP_PITCAIRN:
1677 chip_name = "PITCAIRN";
6458bd4d
AD		 1678 if ((rdev->pdev->revision == 0x81) &&
1679 ((rdev->pdev->device == 0x6810) ||
1680 (rdev->pdev->device == 0x6811)))
861c7fde 1681 new_smc = true;
629bd33c 1682 new_chip_name = "pitcairn";
0f0de06c
AD		 1683 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1684 me_req_size = SI_PM4_UCODE_SIZE * 4;
1685 ce_req_size = SI_CE_UCODE_SIZE * 4;
1686 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1687 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1688 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
a9e61410 1689 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1690 break;
1691 case CHIP_VERDE:
1692 chip_name = "VERDE";
6458bd4d
AD		 1693 if (((rdev->pdev->device == 0x6820) &&
1694 ((rdev->pdev->revision == 0x81) ||
1695 (rdev->pdev->revision == 0x83))) ||
1696 ((rdev->pdev->device == 0x6821) &&
1697 ((rdev->pdev->revision == 0x83) ||
1698 (rdev->pdev->revision == 0x87))) ||
1699 ((rdev->pdev->revision == 0x87) &&
1700 ((rdev->pdev->device == 0x6823) ||
1701 (rdev->pdev->device == 0x682b))))
861c7fde 1702 new_smc = true;
629bd33c 1703 new_chip_name = "verde";
0f0de06c
AD		 1704 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1705 me_req_size = SI_PM4_UCODE_SIZE * 4;
1706 ce_req_size = SI_CE_UCODE_SIZE * 4;
1707 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1708 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1709 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
a9e61410 1710 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
0f0de06c 1711 break;
bcc7f5d2
AD		 1712 case CHIP_OLAND:
1713 chip_name = "OLAND";
6458bd4d
AD		 1714 if (((rdev->pdev->revision == 0x81) &&
1715 ((rdev->pdev->device == 0x6600) ||
1716 (rdev->pdev->device == 0x6604) ||
1717 (rdev->pdev->device == 0x6605) ||
1718 (rdev->pdev->device == 0x6610))) ||
1719 ((rdev->pdev->revision == 0x83) &&
1720 (rdev->pdev->device == 0x6610)))
861c7fde 1721 new_smc = true;
629bd33c 1722 new_chip_name = "oland";
bcc7f5d2
AD		 1723 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1724 me_req_size = SI_PM4_UCODE_SIZE * 4;
1725 ce_req_size = SI_CE_UCODE_SIZE * 4;
1726 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1ebe9280 1727 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1728 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
bcc7f5d2 1729 break;
c04c00b4
AD		 1730 case CHIP_HAINAN:
1731 chip_name = "HAINAN";
6458bd4d
AD		 1732 if (((rdev->pdev->revision == 0x81) &&
1733 (rdev->pdev->device == 0x6660)) ||
1734 ((rdev->pdev->revision == 0x83) &&
1735 ((rdev->pdev->device == 0x6660) ||
1736 (rdev->pdev->device == 0x6663) ||
1737 (rdev->pdev->device == 0x6665) ||
4e6e98b1 1738 (rdev->pdev->device == 0x6667))))
861c7fde 1739 new_smc = true;
4e6e98b1
AD		 1740 else if ((rdev->pdev->revision == 0xc3) &&
1741 (rdev->pdev->device == 0x6665))
1742 banks2_fw = true;
629bd33c 1743 new_chip_name = "hainan";
c04c00b4
AD		 1744 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1745 me_req_size = SI_PM4_UCODE_SIZE * 4;
1746 ce_req_size = SI_CE_UCODE_SIZE * 4;
1747 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1ebe9280 1748 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1749 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
c04c00b4 1750 break;
ce136e15
XC		 1751 default:
1752 BUG();
0f0de06c
AD		 1753 }
1754
ef736d39
AD		 1755 /* this memory configuration requires special firmware */
1756 if (((RREG32(MC_SEQ_MISC0) & 0xff000000) >> 24) == 0x58)
1757 si58_fw = true;
1758
629bd33c 1759 DRM_INFO("Loading %s Microcode\n", new_chip_name);
0f0de06c 1760
629bd33c 1761 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 1762 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
629bd33c
AD		 1763 if (err) {
1764 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1765 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1766 if (err)
1767 goto out;
1768 if (rdev->pfp_fw->size != pfp_req_size) {
7ca85295 1769 pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1770 rdev->pfp_fw->size, fw_name);
1771 err = -EINVAL;
1772 goto out;
1773 }
1774 } else {
1775 err = radeon_ucode_validate(rdev->pfp_fw);
1776 if (err) {
7ca85295 1777 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c
AD		 1778 fw_name);
1779 goto out;
1780 } else {
1781 new_fw++;
1782 }
0f0de06c
AD		 1783 }
1784
629bd33c 1785 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 1786 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
629bd33c
AD		 1787 if (err) {
1788 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1789 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1790 if (err)
1791 goto out;
1792 if (rdev->me_fw->size != me_req_size) {
7ca85295 1793 pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1794 rdev->me_fw->size, fw_name);
1795 err = -EINVAL;
1796 }
1797 } else {
1798 err = radeon_ucode_validate(rdev->me_fw);
1799 if (err) {
7ca85295 1800 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c
AD		 1801 fw_name);
1802 goto out;
1803 } else {
1804 new_fw++;
1805 }
0f0de06c
AD		 1806 }
1807
629bd33c 1808 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 1809 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
629bd33c
AD		 1810 if (err) {
1811 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1812 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1813 if (err)
1814 goto out;
1815 if (rdev->ce_fw->size != ce_req_size) {
7ca85295 1816 pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1817 rdev->ce_fw->size, fw_name);
1818 err = -EINVAL;
1819 }
1820 } else {
1821 err = radeon_ucode_validate(rdev->ce_fw);
1822 if (err) {
7ca85295 1823 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c
AD		 1824 fw_name);
1825 goto out;
1826 } else {
1827 new_fw++;
1828 }
0f0de06c
AD		 1829 }
1830
629bd33c 1831 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 1832 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
629bd33c
AD		 1833 if (err) {
1834 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1835 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1836 if (err)
1837 goto out;
1838 if (rdev->rlc_fw->size != rlc_req_size) {
7ca85295 1839 pr_err("si_rlc: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1840 rdev->rlc_fw->size, fw_name);
1841 err = -EINVAL;
1842 }
1843 } else {
1844 err = radeon_ucode_validate(rdev->rlc_fw);
1845 if (err) {
7ca85295 1846 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c
AD		 1847 fw_name);
1848 goto out;
1849 } else {
1850 new_fw++;
1851 }
0f0de06c
AD		 1852 }
1853
ef736d39
AD		 1854 if (si58_fw)
1855 snprintf(fw_name, sizeof(fw_name), "radeon/si58_mc.bin");
1856 else
1857 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 1858 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1ebe9280 1859 if (err) {
629bd33c 1860 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1ebe9280 1861 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
629bd33c
AD		 1862 if (err) {
1863 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1864 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1865 if (err)
1866 goto out;
1867 }
1868 if ((rdev->mc_fw->size != mc_req_size) &&
1869 (rdev->mc_fw->size != mc2_req_size)) {
7ca85295 1870 pr_err("si_mc: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1871 rdev->mc_fw->size, fw_name);
1872 err = -EINVAL;
1873 }
1874 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1875 } else {
1876 err = radeon_ucode_validate(rdev->mc_fw);
1877 if (err) {
7ca85295 1878 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c 1879 fw_name);
1ebe9280 1880 goto out;
629bd33c
AD		 1881 } else {
1882 new_fw++;
1883 }
1ebe9280 1884 }
0f0de06c 1885
4e6e98b1
AD		 1886 if (banks2_fw)
1887 snprintf(fw_name, sizeof(fw_name), "radeon/banks_k_2_smc.bin");
1888 else if (new_smc)
861c7fde
AD		 1889 snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
1890 else
1891 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
0a168933 1892 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
8a53fa23 1893 if (err) {
629bd33c
AD		 1894 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1895 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1896 if (err) {
7ca85295 1897 pr_err("smc: error loading firmware \"%s\"\n", fw_name);
629bd33c
AD		 1898 release_firmware(rdev->smc_fw);
1899 rdev->smc_fw = NULL;
1900 err = 0;
1901 } else if (rdev->smc_fw->size != smc_req_size) {
7ca85295 1902 pr_err("si_smc: Bogus length %zu in firmware \"%s\"\n",
629bd33c
AD		 1903 rdev->smc_fw->size, fw_name);
1904 err = -EINVAL;
1905 }
1906 } else {
1907 err = radeon_ucode_validate(rdev->smc_fw);
1908 if (err) {
7ca85295 1909 pr_err("si_cp: validation failed for firmware \"%s\"\n",
629bd33c
AD		 1910 fw_name);
1911 goto out;
1912 } else {
1913 new_fw++;
1914 }
a9e61410
AD		 1915 }
1916
629bd33c
AD		 1917 if (new_fw == 0) {
1918 rdev->new_fw = false;
1919 } else if (new_fw < 6) {
7ca85295 1920 pr_err("si_fw: mixing new and old firmware!\n");
629bd33c
AD		 1921 err = -EINVAL;
1922 } else {
1923 rdev->new_fw = true;
1924 }
0f0de06c 1925out:
0f0de06c
AD		 1926 if (err) {
1927 if (err != -EINVAL)
7ca85295 1928 pr_err("si_cp: Failed to load firmware \"%s\"\n",
0f0de06c
AD		 1929 fw_name);
1930 release_firmware(rdev->pfp_fw);
1931 rdev->pfp_fw = NULL;
1932 release_firmware(rdev->me_fw);
1933 rdev->me_fw = NULL;
1934 release_firmware(rdev->ce_fw);
1935 rdev->ce_fw = NULL;
1936 release_firmware(rdev->rlc_fw);
1937 rdev->rlc_fw = NULL;
1938 release_firmware(rdev->mc_fw);
1939 rdev->mc_fw = NULL;
a9e61410
AD		 1940 release_firmware(rdev->smc_fw);
1941 rdev->smc_fw = NULL;
0f0de06c
AD		 1942 }
1943 return err;
1944}
1945
43b3cd99
AD		 1946/* watermark setup */
1947static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1948 struct radeon_crtc *radeon_crtc,
1949 struct drm_display_mode *mode,
1950 struct drm_display_mode *other_mode)
1951{
290d2457
AD		 1952 u32 tmp, buffer_alloc, i;
1953 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
43b3cd99
AD		 1954 /*
1955 * Line Buffer Setup
1956 * There are 3 line buffers, each one shared by 2 display controllers.
1957 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1958 * the display controllers. The paritioning is done via one of four
1959 * preset allocations specified in bits 21:20:
1960 * 0 - half lb
1961 * 2 - whole lb, other crtc must be disabled
1962 */
1963 /* this can get tricky if we have two large displays on a paired group
1964 * of crtcs. Ideally for multiple large displays we'd assign them to
1965 * non-linked crtcs for maximum line buffer allocation.
1966 */
1967 if (radeon_crtc->base.enabled && mode) {
290d2457 1968 if (other_mode) {
43b3cd99 1969 tmp = 0; /* 1/2 */
290d2457
AD		 1970 buffer_alloc = 1;
1971 } else {
43b3cd99 1972 tmp = 2; /* whole */
290d2457
AD		 1973 buffer_alloc = 2;
1974 }
1975 } else {
43b3cd99 1976 tmp = 0;
290d2457
AD		 1977 buffer_alloc = 0;
1978 }
43b3cd99
AD		 1979
1980 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1981 DC_LB_MEMORY_CONFIG(tmp));
1982
290d2457
AD		 1983 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1984 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1985 for (i = 0; i < rdev->usec_timeout; i++) {
1986 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1987 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1988 break;
1989 udelay(1);
1990 }
1991
43b3cd99
AD		 1992 if (radeon_crtc->base.enabled && mode) {
1993 switch (tmp) {
1994 case 0:
1995 default:
1996 return 4096 * 2;
1997 case 2:
1998 return 8192 * 2;
1999 }
2000 }
2001
2002 /* controller not enabled, so no lb used */
2003 return 0;
2004}
2005
ca7db22b 2006static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 2007{
2008 u32 tmp = RREG32(MC_SHARED_CHMAP);
2009
2010 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2011 case 0:
2012 default:
2013 return 1;
2014 case 1:
2015 return 2;
2016 case 2:
2017 return 4;
2018 case 3:
2019 return 8;
2020 case 4:
2021 return 3;
2022 case 5:
2023 return 6;
2024 case 6:
2025 return 10;
2026 case 7:
2027 return 12;
2028 case 8:
2029 return 16;
2030 }
2031}
2032
2033struct dce6_wm_params {
2034 u32 dram_channels; /* number of dram channels */
2035 u32 yclk; /* bandwidth per dram data pin in kHz */
2036 u32 sclk; /* engine clock in kHz */
2037 u32 disp_clk; /* display clock in kHz */
2038 u32 src_width; /* viewport width */
2039 u32 active_time; /* active display time in ns */
2040 u32 blank_time; /* blank time in ns */
2041 bool interlaced; /* mode is interlaced */
2042 fixed20_12 vsc; /* vertical scale ratio */
2043 u32 num_heads; /* number of active crtcs */
2044 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2045 u32 lb_size; /* line buffer allocated to pipe */
2046 u32 vtaps; /* vertical scaler taps */
2047};
2048
2049static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
2050{
2051 /* Calculate raw DRAM Bandwidth */
2052 fixed20_12 dram_efficiency; /* 0.7 */
2053 fixed20_12 yclk, dram_channels, bandwidth;
2054 fixed20_12 a;
2055
2056 a.full = dfixed_const(1000);
2057 yclk.full = dfixed_const(wm->yclk);
2058 yclk.full = dfixed_div(yclk, a);
2059 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2060 a.full = dfixed_const(10);
2061 dram_efficiency.full = dfixed_const(7);
2062 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2063 bandwidth.full = dfixed_mul(dram_channels, yclk);
2064 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2065
2066 return dfixed_trunc(bandwidth);
2067}
2068
2069static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2070{
2071 /* Calculate DRAM Bandwidth and the part allocated to display. */
2072 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2073 fixed20_12 yclk, dram_channels, bandwidth;
2074 fixed20_12 a;
2075
2076 a.full = dfixed_const(1000);
2077 yclk.full = dfixed_const(wm->yclk);
2078 yclk.full = dfixed_div(yclk, a);
2079 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2080 a.full = dfixed_const(10);
2081 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2082 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2083 bandwidth.full = dfixed_mul(dram_channels, yclk);
2084 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2085
2086 return dfixed_trunc(bandwidth);
2087}
2088
2089static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2090{
2091 /* Calculate the display Data return Bandwidth */
2092 fixed20_12 return_efficiency; /* 0.8 */
2093 fixed20_12 sclk, bandwidth;
2094 fixed20_12 a;
2095
2096 a.full = dfixed_const(1000);
2097 sclk.full = dfixed_const(wm->sclk);
2098 sclk.full = dfixed_div(sclk, a);
2099 a.full = dfixed_const(10);
2100 return_efficiency.full = dfixed_const(8);
2101 return_efficiency.full = dfixed_div(return_efficiency, a);
2102 a.full = dfixed_const(32);
2103 bandwidth.full = dfixed_mul(a, sclk);
2104 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2105
2106 return dfixed_trunc(bandwidth);
2107}
2108
2109static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2110{
2111 return 32;
2112}
2113
2114static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2115{
2116 /* Calculate the DMIF Request Bandwidth */
2117 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2118 fixed20_12 disp_clk, sclk, bandwidth;
2119 fixed20_12 a, b1, b2;
2120 u32 min_bandwidth;
2121
2122 a.full = dfixed_const(1000);
2123 disp_clk.full = dfixed_const(wm->disp_clk);
2124 disp_clk.full = dfixed_div(disp_clk, a);
2125 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2126 b1.full = dfixed_mul(a, disp_clk);
2127
2128 a.full = dfixed_const(1000);
2129 sclk.full = dfixed_const(wm->sclk);
2130 sclk.full = dfixed_div(sclk, a);
2131 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2132 b2.full = dfixed_mul(a, sclk);
2133
2134 a.full = dfixed_const(10);
2135 disp_clk_request_efficiency.full = dfixed_const(8);
2136 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2137
2138 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2139
2140 a.full = dfixed_const(min_bandwidth);
2141 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2142
2143 return dfixed_trunc(bandwidth);
2144}
2145
2146static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2147{
2148 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2149 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2150 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2151 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2152
2153 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2154}
2155
2156static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2157{
2158 /* Calculate the display mode Average Bandwidth
2159 * DisplayMode should contain the source and destination dimensions,
2160 * timing, etc.
2161 */
2162 fixed20_12 bpp;
2163 fixed20_12 line_time;
2164 fixed20_12 src_width;
2165 fixed20_12 bandwidth;
2166 fixed20_12 a;
2167
2168 a.full = dfixed_const(1000);
2169 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2170 line_time.full = dfixed_div(line_time, a);
2171 bpp.full = dfixed_const(wm->bytes_per_pixel);
2172 src_width.full = dfixed_const(wm->src_width);
2173 bandwidth.full = dfixed_mul(src_width, bpp);
2174 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2175 bandwidth.full = dfixed_div(bandwidth, line_time);
2176
2177 return dfixed_trunc(bandwidth);
2178}
2179
2180static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2181{
2182 /* First calcualte the latency in ns */
2183 u32 mc_latency = 2000; /* 2000 ns. */
2184 u32 available_bandwidth = dce6_available_bandwidth(wm);
2185 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2186 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2187 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2188 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2189 (wm->num_heads * cursor_line_pair_return_time);
2190 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2191 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2192 u32 tmp, dmif_size = 12288;
2193 fixed20_12 a, b, c;
2194
2195 if (wm->num_heads == 0)
2196 return 0;
2197
2198 a.full = dfixed_const(2);
2199 b.full = dfixed_const(1);
2200 if ((wm->vsc.full > a.full) ||
2201 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2202 (wm->vtaps >= 5) ||
2203 ((wm->vsc.full >= a.full) && wm->interlaced))
2204 max_src_lines_per_dst_line = 4;
2205 else
2206 max_src_lines_per_dst_line = 2;
2207
2208 a.full = dfixed_const(available_bandwidth);
2209 b.full = dfixed_const(wm->num_heads);
2210 a.full = dfixed_div(a, b);
ae45bbc2
MK		 2211 tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
2212 tmp = min(dfixed_trunc(a), tmp);
43b3cd99 2213
ae45bbc2 2214 lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
43b3cd99
AD		 2215
2216 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2217 b.full = dfixed_const(1000);
2218 c.full = dfixed_const(lb_fill_bw);
2219 b.full = dfixed_div(c, b);
2220 a.full = dfixed_div(a, b);
2221 line_fill_time = dfixed_trunc(a);
2222
2223 if (line_fill_time < wm->active_time)
2224 return latency;
2225 else
2226 return latency + (line_fill_time - wm->active_time);
2227
2228}
2229
2230static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2231{
2232 if (dce6_average_bandwidth(wm) <=
2233 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2234 return true;
2235 else
2236 return false;
2237};
2238
2239static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2240{
2241 if (dce6_average_bandwidth(wm) <=
2242 (dce6_available_bandwidth(wm) / wm->num_heads))
2243 return true;
2244 else
2245 return false;
2246};
2247
2248static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2249{
2250 u32 lb_partitions = wm->lb_size / wm->src_width;
2251 u32 line_time = wm->active_time + wm->blank_time;
2252 u32 latency_tolerant_lines;
2253 u32 latency_hiding;
2254 fixed20_12 a;
2255
2256 a.full = dfixed_const(1);
2257 if (wm->vsc.full > a.full)
2258 latency_tolerant_lines = 1;
2259 else {
2260 if (lb_partitions <= (wm->vtaps + 1))
2261 latency_tolerant_lines = 1;
2262 else
2263 latency_tolerant_lines = 2;
2264 }
2265
2266 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2267
2268 if (dce6_latency_watermark(wm) <= latency_hiding)
2269 return true;
2270 else
2271 return false;
2272}
2273
2274static void dce6_program_watermarks(struct radeon_device *rdev,
2275 struct radeon_crtc *radeon_crtc,
2276 u32 lb_size, u32 num_heads)
2277{
2278 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2279 struct dce6_wm_params wm_low, wm_high;
2280 u32 dram_channels;
e6b9a6c8 2281 u32 active_time;
43b3cd99
AD		 2282 u32 line_time = 0;
2283 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2284 u32 priority_a_mark = 0, priority_b_mark = 0;
2285 u32 priority_a_cnt = PRIORITY_OFF;
2286 u32 priority_b_cnt = PRIORITY_OFF;
2287 u32 tmp, arb_control3;
2288 fixed20_12 a, b, c;
2289
2290 if (radeon_crtc->base.enabled && num_heads && mode) {
55f61a04
MK		 2291 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2292 (u32)mode->clock);
2293 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2294 (u32)mode->clock);
2295 line_time = min(line_time, (u32)65535);
43b3cd99
AD		 2296 priority_a_cnt = 0;
2297 priority_b_cnt = 0;
2298
ca7db22b 2299 if (rdev->family == CHIP_ARUBA)
c696e53f 2300 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2301 else
c696e53f
AD		 2302 dram_channels = si_get_number_of_dram_channels(rdev);
2303
2304 /* watermark for high clocks */
2305 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2306 wm_high.yclk =
2307 radeon_dpm_get_mclk(rdev, false) * 10;
2308 wm_high.sclk =
2309 radeon_dpm_get_sclk(rdev, false) * 10;
2310 } else {
2311 wm_high.yclk = rdev->pm.current_mclk * 10;
2312 wm_high.sclk = rdev->pm.current_sclk * 10;
2313 }
2314
2315 wm_high.disp_clk = mode->clock;
2316 wm_high.src_width = mode->crtc_hdisplay;
e6b9a6c8 2317 wm_high.active_time = active_time;
c696e53f
AD		 2318 wm_high.blank_time = line_time - wm_high.active_time;
2319 wm_high.interlaced = false;
2320 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2321 wm_high.interlaced = true;
2322 wm_high.vsc = radeon_crtc->vsc;
2323 wm_high.vtaps = 1;
2324 if (radeon_crtc->rmx_type != RMX_OFF)
2325 wm_high.vtaps = 2;
2326 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2327 wm_high.lb_size = lb_size;
2328 wm_high.dram_channels = dram_channels;
2329 wm_high.num_heads = num_heads;
2330
2331 /* watermark for low clocks */
2332 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2333 wm_low.yclk =
2334 radeon_dpm_get_mclk(rdev, true) * 10;
2335 wm_low.sclk =
2336 radeon_dpm_get_sclk(rdev, true) * 10;
2337 } else {
2338 wm_low.yclk = rdev->pm.current_mclk * 10;
2339 wm_low.sclk = rdev->pm.current_sclk * 10;
2340 }
2341
2342 wm_low.disp_clk = mode->clock;
2343 wm_low.src_width = mode->crtc_hdisplay;
e6b9a6c8 2344 wm_low.active_time = active_time;
c696e53f
AD		 2345 wm_low.blank_time = line_time - wm_low.active_time;
2346 wm_low.interlaced = false;
2347 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2348 wm_low.interlaced = true;
2349 wm_low.vsc = radeon_crtc->vsc;
2350 wm_low.vtaps = 1;
2351 if (radeon_crtc->rmx_type != RMX_OFF)
2352 wm_low.vtaps = 2;
2353 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2354 wm_low.lb_size = lb_size;
2355 wm_low.dram_channels = dram_channels;
2356 wm_low.num_heads = num_heads;
43b3cd99
AD		 2357
2358 /* set for high clocks */
c696e53f 2359 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2360 /* set for low clocks */
c696e53f 2361 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2362
2363 /* possibly force display priority to high */
2364 /* should really do this at mode validation time... */
c696e53f
AD		 2365 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2366 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2367 !dce6_check_latency_hiding(&wm_high) ||
2368 (rdev->disp_priority == 2)) {
2369 DRM_DEBUG_KMS("force priority to high\n");
2370 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2371 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2372 }
2373 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2374 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2375 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2376 (rdev->disp_priority == 2)) {
2377 DRM_DEBUG_KMS("force priority to high\n");
2378 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2379 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2380 }
2381
2382 a.full = dfixed_const(1000);
2383 b.full = dfixed_const(mode->clock);
2384 b.full = dfixed_div(b, a);
2385 c.full = dfixed_const(latency_watermark_a);
2386 c.full = dfixed_mul(c, b);
2387 c.full = dfixed_mul(c, radeon_crtc->hsc);
2388 c.full = dfixed_div(c, a);
2389 a.full = dfixed_const(16);
2390 c.full = dfixed_div(c, a);
2391 priority_a_mark = dfixed_trunc(c);
2392 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2393
2394 a.full = dfixed_const(1000);
2395 b.full = dfixed_const(mode->clock);
2396 b.full = dfixed_div(b, a);
2397 c.full = dfixed_const(latency_watermark_b);
2398 c.full = dfixed_mul(c, b);
2399 c.full = dfixed_mul(c, radeon_crtc->hsc);
2400 c.full = dfixed_div(c, a);
2401 a.full = dfixed_const(16);
2402 c.full = dfixed_div(c, a);
2403 priority_b_mark = dfixed_trunc(c);
2404 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
5b5561b3
MK		 2405
2406 /* Save number of lines the linebuffer leads before the scanout */
2407 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
43b3cd99
AD		 2408 }
2409
2410 /* select wm A */
2411 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2412 tmp = arb_control3;
2413 tmp &= ~LATENCY_WATERMARK_MASK(3);
2414 tmp |= LATENCY_WATERMARK_MASK(1);
2415 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2416 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2417 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2418 LATENCY_HIGH_WATERMARK(line_time)));
2419 /* select wm B */
2420 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2421 tmp &= ~LATENCY_WATERMARK_MASK(3);
2422 tmp |= LATENCY_WATERMARK_MASK(2);
2423 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2424 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2425 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2426 LATENCY_HIGH_WATERMARK(line_time)));
2427 /* restore original selection */
2428 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2429
2430 /* write the priority marks */
2431 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2432 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2433
7178d2a6
AD		 2434 /* save values for DPM */
2435 radeon_crtc->line_time = line_time;
2436 radeon_crtc->wm_high = latency_watermark_a;
2437 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2438}
2439
2440void dce6_bandwidth_update(struct radeon_device *rdev)
2441{
2442 struct drm_display_mode *mode0 = NULL;
2443 struct drm_display_mode *mode1 = NULL;
2444 u32 num_heads = 0, lb_size;
2445 int i;
2446
8efe82ca
AD		 2447 if (!rdev->mode_info.mode_config_initialized)
2448 return;
2449
43b3cd99
AD		 2450 radeon_update_display_priority(rdev);
2451
2452 for (i = 0; i < rdev->num_crtc; i++) {
2453 if (rdev->mode_info.crtcs[i]->base.enabled)
2454 num_heads++;
2455 }
2456 for (i = 0; i < rdev->num_crtc; i += 2) {
2457 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2458 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2459 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2460 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2461 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2462 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2463 }
2464}
2465
0a96d72b
AD		 2466/*
2467 * Core functions
2468 */
0a96d72b
AD		 2469static void si_tiling_mode_table_init(struct radeon_device *rdev)
2470{
102534b0
JP		 2471 u32 *tile = rdev->config.si.tile_mode_array;
2472 const u32 num_tile_mode_states =
2473 ARRAY_SIZE(rdev->config.si.tile_mode_array);
2474 u32 reg_offset, split_equal_to_row_size;
0a96d72b
AD		 2475
2476 switch (rdev->config.si.mem_row_size_in_kb) {
2477 case 1:
2478 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2479 break;
2480 case 2:
2481 default:
2482 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2483 break;
2484 case 4:
2485 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2486 break;
2487 }
2488
102534b0
JP		 2489 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2490 tile[reg_offset] = 0;
2491
2492 switch(rdev->family) {
2493 case CHIP_TAHITI:
2494 case CHIP_PITCAIRN:
2495 /* non-AA compressed depth or any compressed stencil */
2496 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2497 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2498 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2499 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2500 NUM_BANKS(ADDR_SURF_16_BANK) |
2501 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2502 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2503 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2504 /* 2xAA/4xAA compressed depth only */
2505 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2506 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2507 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2508 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2509 NUM_BANKS(ADDR_SURF_16_BANK) |
2510 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2511 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2512 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2513 /* 8xAA compressed depth only */
2514 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2515 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2516 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2517 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2518 NUM_BANKS(ADDR_SURF_16_BANK) |
2519 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2520 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2521 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2522 /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2523 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2525 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2526 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2527 NUM_BANKS(ADDR_SURF_16_BANK) |
2528 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2529 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2530 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2531 /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2532 tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2533 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2534 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2535 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2536 NUM_BANKS(ADDR_SURF_16_BANK) |
2537 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2538 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2539 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2540 /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2541 tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2542 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2543 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2544 TILE_SPLIT(split_equal_to_row_size) |
2545 NUM_BANKS(ADDR_SURF_16_BANK) |
2546 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2547 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2548 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2549 /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2550 tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2551 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2552 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2553 TILE_SPLIT(split_equal_to_row_size) |
2554 NUM_BANKS(ADDR_SURF_16_BANK) |
2555 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2556 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2557 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2558 /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2559 tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2560 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2562 TILE_SPLIT(split_equal_to_row_size) |
2563 NUM_BANKS(ADDR_SURF_16_BANK) |
2564 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2565 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2566 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2567 /* 1D and 1D Array Surfaces */
2568 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2569 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2571 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2572 NUM_BANKS(ADDR_SURF_16_BANK) |
2573 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2574 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2575 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2576 /* Displayable maps. */
2577 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2578 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2579 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2580 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2581 NUM_BANKS(ADDR_SURF_16_BANK) |
2582 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2583 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2584 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2585 /* Display 8bpp. */
2586 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2587 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2588 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2589 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2590 NUM_BANKS(ADDR_SURF_16_BANK) |
2591 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2593 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2594 /* Display 16bpp. */
2595 tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2596 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2598 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2599 NUM_BANKS(ADDR_SURF_16_BANK) |
2600 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2601 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2602 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2603 /* Display 32bpp. */
2604 tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2607 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2608 NUM_BANKS(ADDR_SURF_16_BANK) |
2609 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2610 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2611 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2612 /* Thin. */
2613 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2614 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2615 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2616 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2617 NUM_BANKS(ADDR_SURF_16_BANK) |
2618 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2620 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2621 /* Thin 8 bpp. */
2622 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2623 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2624 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2625 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2626 NUM_BANKS(ADDR_SURF_16_BANK) |
2627 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2630 /* Thin 16 bpp. */
2631 tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2632 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2633 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2634 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2635 NUM_BANKS(ADDR_SURF_16_BANK) |
2636 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2638 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2639 /* Thin 32 bpp. */
2640 tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2643 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2644 NUM_BANKS(ADDR_SURF_16_BANK) |
2645 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2646 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2647 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2648 /* Thin 64 bpp. */
2649 tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2650 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2651 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2652 TILE_SPLIT(split_equal_to_row_size) |
2653 NUM_BANKS(ADDR_SURF_16_BANK) |
2654 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2657 /* 8 bpp PRT. */
2658 tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2659 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2660 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2661 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2662 NUM_BANKS(ADDR_SURF_16_BANK) |
2663 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2664 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2665 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2666 /* 16 bpp PRT */
2667 tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2668 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2669 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2670 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2671 NUM_BANKS(ADDR_SURF_16_BANK) |
2672 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2674 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2675 /* 32 bpp PRT */
2676 tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2677 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2678 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2679 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2680 NUM_BANKS(ADDR_SURF_16_BANK) |
2681 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2682 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2683 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2684 /* 64 bpp PRT */
2685 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2686 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2687 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2688 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2689 NUM_BANKS(ADDR_SURF_16_BANK) |
2690 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2693 /* 128 bpp PRT */
2694 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2695 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2696 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2697 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2698 NUM_BANKS(ADDR_SURF_8_BANK) |
2699 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2700 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2701 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2702
2703 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2704 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2705 break;
2706
2707 case CHIP_VERDE:
2708 case CHIP_OLAND:
2709 case CHIP_HAINAN:
2710 /* non-AA compressed depth or any compressed stencil */
2711 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2712 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2713 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2714 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2715 NUM_BANKS(ADDR_SURF_16_BANK) |
2716 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2719 /* 2xAA/4xAA compressed depth only */
2720 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2721 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2722 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2723 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2724 NUM_BANKS(ADDR_SURF_16_BANK) |
2725 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2728 /* 8xAA compressed depth only */
2729 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2730 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2731 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2732 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2733 NUM_BANKS(ADDR_SURF_16_BANK) |
2734 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2735 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2736 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2737 /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2738 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2739 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2740 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2741 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2742 NUM_BANKS(ADDR_SURF_16_BANK) |
2743 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2746 /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2747 tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2748 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2749 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2750 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2751 NUM_BANKS(ADDR_SURF_16_BANK) |
2752 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2753 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2754 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2755 /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2756 tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2757 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2758 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759 TILE_SPLIT(split_equal_to_row_size) |
2760 NUM_BANKS(ADDR_SURF_16_BANK) |
2761 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2764 /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2765 tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2766 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2767 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2768 TILE_SPLIT(split_equal_to_row_size) |
2769 NUM_BANKS(ADDR_SURF_16_BANK) |
2770 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2771 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2772 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2773 /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2774 tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2775 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2776 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2777 TILE_SPLIT(split_equal_to_row_size) |
2778 NUM_BANKS(ADDR_SURF_16_BANK) |
2779 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2780 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2781 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2782 /* 1D and 1D Array Surfaces */
2783 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2784 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2785 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2786 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2787 NUM_BANKS(ADDR_SURF_16_BANK) |
2788 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2789 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2790 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2791 /* Displayable maps. */
2792 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2793 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2794 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2795 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2796 NUM_BANKS(ADDR_SURF_16_BANK) |
2797 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2798 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2799 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2800 /* Display 8bpp. */
2801 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2802 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2803 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2804 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2805 NUM_BANKS(ADDR_SURF_16_BANK) |
2806 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2807 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2808 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2809 /* Display 16bpp. */
2810 tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2811 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2812 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2813 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2814 NUM_BANKS(ADDR_SURF_16_BANK) |
2815 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2816 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2817 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2818 /* Display 32bpp. */
2819 tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2822 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2823 NUM_BANKS(ADDR_SURF_16_BANK) |
2824 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2825 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2826 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2827 /* Thin. */
2828 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2829 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2830 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2831 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2832 NUM_BANKS(ADDR_SURF_16_BANK) |
2833 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2834 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2835 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2836 /* Thin 8 bpp. */
2837 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2840 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2841 NUM_BANKS(ADDR_SURF_16_BANK) |
2842 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2843 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2844 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2845 /* Thin 16 bpp. */
2846 tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2847 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2848 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2849 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2850 NUM_BANKS(ADDR_SURF_16_BANK) |
2851 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2852 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2853 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2854 /* Thin 32 bpp. */
2855 tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2856 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2857 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2858 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2859 NUM_BANKS(ADDR_SURF_16_BANK) |
2860 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2861 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2862 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2863 /* Thin 64 bpp. */
2864 tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2865 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2866 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2867 TILE_SPLIT(split_equal_to_row_size) |
2868 NUM_BANKS(ADDR_SURF_16_BANK) |
2869 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2870 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2871 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2872 /* 8 bpp PRT. */
2873 tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2874 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2875 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2876 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2877 NUM_BANKS(ADDR_SURF_16_BANK) |
2878 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2879 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2880 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2881 /* 16 bpp PRT */
2882 tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2883 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2884 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2885 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2886 NUM_BANKS(ADDR_SURF_16_BANK) |
2887 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2888 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2889 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2890 /* 32 bpp PRT */
2891 tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2895 NUM_BANKS(ADDR_SURF_16_BANK) |
2896 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2897 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2898 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2899 /* 64 bpp PRT */
2900 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2901 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2902 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2903 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2904 NUM_BANKS(ADDR_SURF_16_BANK) |
2905 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2906 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2907 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2908 /* 128 bpp PRT */
2909 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2910 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2911 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2912 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2913 NUM_BANKS(ADDR_SURF_8_BANK) |
2914 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2917
2918 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2919 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2920 break;
2921
2922 default:
0a96d72b 2923 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
102534b0 2924 }
0a96d72b
AD		 2925}
2926
1a8ca750
AD		 2927static void si_select_se_sh(struct radeon_device *rdev,
2928 u32 se_num, u32 sh_num)
2929{
2930 u32 data = INSTANCE_BROADCAST_WRITES;
2931
2932 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2933 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2934 else if (se_num == 0xffffffff)
2935 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2936 else if (sh_num == 0xffffffff)
2937 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2938 else
2939 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2940 WREG32(GRBM_GFX_INDEX, data);
2941}
2942
2943static u32 si_create_bitmask(u32 bit_width)
2944{
2945 u32 i, mask = 0;
2946
2947 for (i = 0; i < bit_width; i++) {
2948 mask <<= 1;
2949 mask |= 1;
2950 }
2951 return mask;
2952}
2953
2954static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2955{
2956 u32 data, mask;
2957
2958 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2959 if (data & 1)
2960 data &= INACTIVE_CUS_MASK;
2961 else
2962 data = 0;
2963 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2964
2965 data >>= INACTIVE_CUS_SHIFT;
2966
2967 mask = si_create_bitmask(cu_per_sh);
2968
2969 return ~data & mask;
2970}
2971
2972static void si_setup_spi(struct radeon_device *rdev,
2973 u32 se_num, u32 sh_per_se,
2974 u32 cu_per_sh)
2975{
2976 int i, j, k;
2977 u32 data, mask, active_cu;
2978
2979 for (i = 0; i < se_num; i++) {
2980 for (j = 0; j < sh_per_se; j++) {
2981 si_select_se_sh(rdev, i, j);
2982 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2983 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2984
2985 mask = 1;
2986 for (k = 0; k < 16; k++) {
2987 mask <<= k;
2988 if (active_cu & mask) {
2989 data &= ~mask;
2990 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2991 break;
2992 }
2993 }
2994 }
2995 }
2996 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2997}
2998
2999static u32 si_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3000 u32 max_rb_num_per_se,
1a8ca750
AD		 3001 u32 sh_per_se)
3002{
3003 u32 data, mask;
3004
3005 data = RREG32(CC_RB_BACKEND_DISABLE);
3006 if (data & 1)
3007 data &= BACKEND_DISABLE_MASK;
3008 else
3009 data = 0;
3010 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3011
3012 data >>= BACKEND_DISABLE_SHIFT;
3013
9fadb352 3014 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
1a8ca750
AD		 3015
3016 return data & mask;
3017}
3018
3019static void si_setup_rb(struct radeon_device *rdev,
3020 u32 se_num, u32 sh_per_se,
9fadb352 3021 u32 max_rb_num_per_se)
1a8ca750
AD		 3022{
3023 int i, j;
3024 u32 data, mask;
3025 u32 disabled_rbs = 0;
3026 u32 enabled_rbs = 0;
3027
3028 for (i = 0; i < se_num; i++) {
3029 for (j = 0; j < sh_per_se; j++) {
3030 si_select_se_sh(rdev, i, j);
9fadb352 3031 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
1a8ca750
AD		 3032 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3033 }
3034 }
3035 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3036
3037 mask = 1;
9fadb352 3038 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
1a8ca750
AD		 3039 if (!(disabled_rbs & mask))
3040 enabled_rbs |= mask;
3041 mask <<= 1;
3042 }
3043
439a1cff
MO		 3044 rdev->config.si.backend_enable_mask = enabled_rbs;
3045
1a8ca750
AD		 3046 for (i = 0; i < se_num; i++) {
3047 si_select_se_sh(rdev, i, 0xffffffff);
3048 data = 0;
3049 for (j = 0; j < sh_per_se; j++) {
3050 switch (enabled_rbs & 3) {
3051 case 1:
3052 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3053 break;
3054 case 2:
3055 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3056 break;
3057 case 3:
3058 default:
3059 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3060 break;
3061 }
3062 enabled_rbs >>= 2;
3063 }
3064 WREG32(PA_SC_RASTER_CONFIG, data);
3065 }
3066 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3067}
3068
0a96d72b
AD		 3069static void si_gpu_init(struct radeon_device *rdev)
3070{
0a96d72b 3071 u32 gb_addr_config = 0;
c317fcbb 3072 u32 mc_arb_ramcfg;
0a96d72b 3073 u32 sx_debug_1;
0a96d72b
AD		 3074 u32 hdp_host_path_cntl;
3075 u32 tmp;
6101b3ae 3076 int i, j;
0a96d72b
AD		 3077
3078 switch (rdev->family) {
3079 case CHIP_TAHITI:
3080 rdev->config.si.max_shader_engines = 2;
0a96d72b 3081 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 3082 rdev->config.si.max_cu_per_sh = 8;
3083 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 3084 rdev->config.si.max_backends_per_se = 4;
3085 rdev->config.si.max_texture_channel_caches = 12;
3086 rdev->config.si.max_gprs = 256;
3087 rdev->config.si.max_gs_threads = 32;
3088 rdev->config.si.max_hw_contexts = 8;
3089
3090 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3091 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3092 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3093 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 3094 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 3095 break;
3096 case CHIP_PITCAIRN:
3097 rdev->config.si.max_shader_engines = 2;
0a96d72b 3098 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 3099 rdev->config.si.max_cu_per_sh = 5;
3100 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 3101 rdev->config.si.max_backends_per_se = 4;
3102 rdev->config.si.max_texture_channel_caches = 8;
3103 rdev->config.si.max_gprs = 256;
3104 rdev->config.si.max_gs_threads = 32;
3105 rdev->config.si.max_hw_contexts = 8;
3106
3107 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3108 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3109 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3110 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 3111 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 3112 break;
3113 case CHIP_VERDE:
3114 default:
3115 rdev->config.si.max_shader_engines = 1;
0a96d72b 3116 rdev->config.si.max_tile_pipes = 4;
468ef1a5 3117 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 3118 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 3119 rdev->config.si.max_backends_per_se = 4;
3120 rdev->config.si.max_texture_channel_caches = 4;
3121 rdev->config.si.max_gprs = 256;
3122 rdev->config.si.max_gs_threads = 32;
3123 rdev->config.si.max_hw_contexts = 8;
3124
d0ae7fcc
AD		 3125 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3126 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3127 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3128 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3129 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3130 break;
3131 case CHIP_OLAND:
3132 rdev->config.si.max_shader_engines = 1;
3133 rdev->config.si.max_tile_pipes = 4;
3134 rdev->config.si.max_cu_per_sh = 6;
3135 rdev->config.si.max_sh_per_se = 1;
3136 rdev->config.si.max_backends_per_se = 2;
3137 rdev->config.si.max_texture_channel_caches = 4;
3138 rdev->config.si.max_gprs = 256;
3139 rdev->config.si.max_gs_threads = 16;
3140 rdev->config.si.max_hw_contexts = 8;
3141
0a96d72b
AD		 3142 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3143 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3144 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3145 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 3146 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 3147 break;
8b02859d
AD		 3148 case CHIP_HAINAN:
3149 rdev->config.si.max_shader_engines = 1;
3150 rdev->config.si.max_tile_pipes = 4;
3151 rdev->config.si.max_cu_per_sh = 5;
3152 rdev->config.si.max_sh_per_se = 1;
3153 rdev->config.si.max_backends_per_se = 1;
3154 rdev->config.si.max_texture_channel_caches = 2;
3155 rdev->config.si.max_gprs = 256;
3156 rdev->config.si.max_gs_threads = 16;
3157 rdev->config.si.max_hw_contexts = 8;
3158
3159 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3160 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3161 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3162 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3163 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3164 break;
0a96d72b
AD		 3165 }
3166
3167 /* Initialize HDP */
3168 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3169 WREG32((0x2c14 + j), 0x00000000);
3170 WREG32((0x2c18 + j), 0x00000000);
3171 WREG32((0x2c1c + j), 0x00000000);
3172 WREG32((0x2c20 + j), 0x00000000);
3173 WREG32((0x2c24 + j), 0x00000000);
3174 }
3175
3176 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
18ad01ef
CK		 3177 WREG32(SRBM_INT_CNTL, 1);
3178 WREG32(SRBM_INT_ACK, 1);
0a96d72b
AD		 3179
3180 evergreen_fix_pci_max_read_req_size(rdev);
3181
3182 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3183
c317fcbb 3184 RREG32(MC_SHARED_CHMAP);
0a96d72b
AD		 3185 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3186
0a96d72b 3187 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 3188 rdev->config.si.mem_max_burst_length_bytes = 256;
3189 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3190 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3191 if (rdev->config.si.mem_row_size_in_kb > 4)
3192 rdev->config.si.mem_row_size_in_kb = 4;
3193 /* XXX use MC settings? */
3194 rdev->config.si.shader_engine_tile_size = 32;
3195 rdev->config.si.num_gpus = 1;
3196 rdev->config.si.multi_gpu_tile_size = 64;
3197
1a8ca750
AD		 3198 /* fix up row size */
3199 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 3200 switch (rdev->config.si.mem_row_size_in_kb) {
3201 case 1:
3202 default:
3203 gb_addr_config |= ROW_SIZE(0);
3204 break;
3205 case 2:
3206 gb_addr_config |= ROW_SIZE(1);
3207 break;
3208 case 4:
3209 gb_addr_config |= ROW_SIZE(2);
3210 break;
3211 }
3212
0a96d72b
AD		 3213 /* setup tiling info dword. gb_addr_config is not adequate since it does
3214 * not have bank info, so create a custom tiling dword.
3215 * bits 3:0 num_pipes
3216 * bits 7:4 num_banks
3217 * bits 11:8 group_size
3218 * bits 15:12 row_size
3219 */
3220 rdev->config.si.tile_config = 0;
3221 switch (rdev->config.si.num_tile_pipes) {
3222 case 1:
3223 rdev->config.si.tile_config |= (0 << 0);
3224 break;
3225 case 2:
3226 rdev->config.si.tile_config |= (1 << 0);
3227 break;
3228 case 4:
3229 rdev->config.si.tile_config |= (2 << 0);
3230 break;
3231 case 8:
3232 default:
3233 /* XXX what about 12? */
3234 rdev->config.si.tile_config |= (3 << 0);
3235 break;
3d581b11 3236 }
dca571a6
CK		 3237 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3238 case 0: /* four banks */
1a8ca750 3239 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3240 break;
3241 case 1: /* eight banks */
3242 rdev->config.si.tile_config |= 1 << 4;
3243 break;
3244 case 2: /* sixteen banks */
3245 default:
3246 rdev->config.si.tile_config |= 2 << 4;
3247 break;
3248 }
0a96d72b
AD		 3249 rdev->config.si.tile_config |=
3250 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3251 rdev->config.si.tile_config |=
3252 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3253
0a96d72b
AD		 3254 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3255 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3256 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3257 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3258 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3259 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3260 if (rdev->has_uvd) {
3261 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3262 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3263 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3264 }
0a96d72b 3265
1a8ca750 3266 si_tiling_mode_table_init(rdev);
0a96d72b 3267
1a8ca750
AD		 3268 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3269 rdev->config.si.max_sh_per_se,
3270 rdev->config.si.max_backends_per_se);
0a96d72b 3271
1a8ca750
AD		 3272 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3273 rdev->config.si.max_sh_per_se,
3274 rdev->config.si.max_cu_per_sh);
0a96d72b 3275
52da51f0 3276 rdev->config.si.active_cus = 0;
65fcf668
AD		 3277 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3278 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
6101b3ae
AD		 3279 rdev->config.si.active_cus +=
3280 hweight32(si_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3281 }
3282 }
0a96d72b
AD		 3283
3284 /* set HW defaults for 3D engine */
3285 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3286 ROQ_IB2_START(0x2b)));
3287 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3288
3289 sx_debug_1 = RREG32(SX_DEBUG_1);
3290 WREG32(SX_DEBUG_1, sx_debug_1);
3291
3292 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3293
3294 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3295 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3296 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3297 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3298
3299 WREG32(VGT_NUM_INSTANCES, 1);
3300
3301 WREG32(CP_PERFMON_CNTL, 0);
3302
3303 WREG32(SQ_CONFIG, 0);
3304
3305 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3306 FORCE_EOV_MAX_REZ_CNT(255)));
3307
3308 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3309 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3310
3311 WREG32(VGT_GS_VERTEX_REUSE, 16);
3312 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3313
3314 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3315 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3316 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3317 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3318 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3319 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3320 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3321 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3322
3323 tmp = RREG32(HDP_MISC_CNTL);
3324 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3325 WREG32(HDP_MISC_CNTL, tmp);
3326
3327 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3328 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3329
3330 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3331
3332 udelay(50);
3333}
c476dde2 3334
2ece2e8b
AD		 3335/*
3336 * GPU scratch registers helpers function.
3337 */
3338static void si_scratch_init(struct radeon_device *rdev)
3339{
3340 int i;
3341
3342 rdev->scratch.num_reg = 7;
3343 rdev->scratch.reg_base = SCRATCH_REG0;
3344 for (i = 0; i < rdev->scratch.num_reg; i++) {
3345 rdev->scratch.free[i] = true;
3346 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3347 }
3348}
3349
3350void si_fence_ring_emit(struct radeon_device *rdev,
3351 struct radeon_fence *fence)
3352{
3353 struct radeon_ring *ring = &rdev->ring[fence->ring];
3354 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3355
3356 /* flush read cache over gart */
3357 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3358 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3359 radeon_ring_write(ring, 0);
3360 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3361 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3362 PACKET3_TC_ACTION_ENA |
3363 PACKET3_SH_KCACHE_ACTION_ENA |
3364 PACKET3_SH_ICACHE_ACTION_ENA);
3365 radeon_ring_write(ring, 0xFFFFFFFF);
3366 radeon_ring_write(ring, 0);
3367 radeon_ring_write(ring, 10); /* poll interval */
3368 /* EVENT_WRITE_EOP - flush caches, send int */
3369 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3370 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
5e167cdb 3371 radeon_ring_write(ring, lower_32_bits(addr));
2ece2e8b
AD		 3372 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3373 radeon_ring_write(ring, fence->seq);
3374 radeon_ring_write(ring, 0);
3375}
3376
3377/*
3378 * IB stuff
3379 */
3380void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3381{
876dc9f3 3382 struct radeon_ring *ring = &rdev->ring[ib->ring];
7c42bc1a 3383 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
2ece2e8b
AD		 3384 u32 header;
3385
a85a7da4
AD		 3386 if (ib->is_const_ib) {
3387 /* set switch buffer packet before const IB */
3388 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3389 radeon_ring_write(ring, 0);
45df6803 3390
2ece2e8b 3391 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3392 } else {
89d35807 3393 u32 next_rptr;
a85a7da4 3394 if (ring->rptr_save_reg) {
89d35807 3395 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3396 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3397 radeon_ring_write(ring, ((ring->rptr_save_reg -
3398 PACKET3_SET_CONFIG_REG_START) >> 2));
3399 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3400 } else if (rdev->wb.enabled) {
3401 next_rptr = ring->wptr + 5 + 4 + 8;
3402 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3403 radeon_ring_write(ring, (1 << 8));
3404 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 3405 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
89d35807 3406 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3407 }
3408
2ece2e8b 3409 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3410 }
2ece2e8b
AD		 3411
3412 radeon_ring_write(ring, header);
3413 radeon_ring_write(ring,
3414#ifdef __BIG_ENDIAN
3415 (2 << 0) |
3416#endif
3417 (ib->gpu_addr & 0xFFFFFFFC));
3418 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
7c42bc1a 3419 radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
2ece2e8b 3420
a85a7da4
AD		 3421 if (!ib->is_const_ib) {
3422 /* flush read cache over gart for this vmid */
3423 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3424 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
7c42bc1a 3425 radeon_ring_write(ring, vm_id);
a85a7da4
AD		 3426 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3427 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3428 PACKET3_TC_ACTION_ENA |
3429 PACKET3_SH_KCACHE_ACTION_ENA |
3430 PACKET3_SH_ICACHE_ACTION_ENA);
3431 radeon_ring_write(ring, 0xFFFFFFFF);
3432 radeon_ring_write(ring, 0);
3433 radeon_ring_write(ring, 10); /* poll interval */
3434 }
2ece2e8b
AD		 3435}
3436
48c0c902
AD		 3437/*
3438 * CP.
3439 */
3440static void si_cp_enable(struct radeon_device *rdev, bool enable)
3441{
3442 if (enable)
3443 WREG32(CP_ME_CNTL, 0);
3444 else {
50efa51a
AD		 3445 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3446 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
48c0c902
AD		 3447 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3448 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3449 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3450 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3451 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3452 }
3453 udelay(50);
3454}
3455
3456static int si_cp_load_microcode(struct radeon_device *rdev)
3457{
48c0c902
AD		 3458 int i;
3459
629bd33c 3460 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
48c0c902
AD		 3461 return -EINVAL;
3462
3463 si_cp_enable(rdev, false);
3464
629bd33c
AD		 3465 if (rdev->new_fw) {
3466 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3467 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3468 const struct gfx_firmware_header_v1_0 *ce_hdr =
3469 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3470 const struct gfx_firmware_header_v1_0 *me_hdr =
3471 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3472 const __le32 *fw_data;
3473 u32 fw_size;
3474
3475 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3476 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3477 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3478
3479 /* PFP */
3480 fw_data = (const __le32 *)
3481 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3482 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3483 WREG32(CP_PFP_UCODE_ADDR, 0);
3484 for (i = 0; i < fw_size; i++)
3485 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3486 WREG32(CP_PFP_UCODE_ADDR, 0);
3487
3488 /* CE */
3489 fw_data = (const __le32 *)
3490 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3491 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3492 WREG32(CP_CE_UCODE_ADDR, 0);
3493 for (i = 0; i < fw_size; i++)
3494 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3495 WREG32(CP_CE_UCODE_ADDR, 0);
3496
3497 /* ME */
3498 fw_data = (const __be32 *)
3499 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3500 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3501 WREG32(CP_ME_RAM_WADDR, 0);
3502 for (i = 0; i < fw_size; i++)
3503 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3504 WREG32(CP_ME_RAM_WADDR, 0);
3505 } else {
3506 const __be32 *fw_data;
3507
3508 /* PFP */
3509 fw_data = (const __be32 *)rdev->pfp_fw->data;
3510 WREG32(CP_PFP_UCODE_ADDR, 0);
3511 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3512 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3513 WREG32(CP_PFP_UCODE_ADDR, 0);
3514
3515 /* CE */
3516 fw_data = (const __be32 *)rdev->ce_fw->data;
3517 WREG32(CP_CE_UCODE_ADDR, 0);
3518 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3519 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3520 WREG32(CP_CE_UCODE_ADDR, 0);
3521
3522 /* ME */
3523 fw_data = (const __be32 *)rdev->me_fw->data;
3524 WREG32(CP_ME_RAM_WADDR, 0);
3525 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3526 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3527 WREG32(CP_ME_RAM_WADDR, 0);
3528 }
48c0c902
AD		 3529
3530 WREG32(CP_PFP_UCODE_ADDR, 0);
3531 WREG32(CP_CE_UCODE_ADDR, 0);
3532 WREG32(CP_ME_RAM_WADDR, 0);
3533 WREG32(CP_ME_RAM_RADDR, 0);
3534 return 0;
3535}
3536
3537static int si_cp_start(struct radeon_device *rdev)
3538{
3539 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3540 int r, i;
3541
3542 r = radeon_ring_lock(rdev, ring, 7 + 4);
3543 if (r) {
3544 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3545 return r;
3546 }
3547 /* init the CP */
3548 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3549 radeon_ring_write(ring, 0x1);
3550 radeon_ring_write(ring, 0x0);
3551 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3552 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3553 radeon_ring_write(ring, 0);
3554 radeon_ring_write(ring, 0);
3555
3556 /* init the CE partitions */
3557 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3558 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3559 radeon_ring_write(ring, 0xc000);
3560 radeon_ring_write(ring, 0xe000);
1538a9e0 3561 radeon_ring_unlock_commit(rdev, ring, false);
48c0c902
AD		 3562
3563 si_cp_enable(rdev, true);
3564
3565 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3566 if (r) {
3567 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3568 return r;
3569 }
3570
3571 /* setup clear context state */
3572 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3573 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3574
3575 for (i = 0; i < si_default_size; i++)
3576 radeon_ring_write(ring, si_default_state[i]);
3577
3578 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3579 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3580
3581 /* set clear context state */
3582 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3583 radeon_ring_write(ring, 0);
3584
3585 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3586 radeon_ring_write(ring, 0x00000316);
3587 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3588 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3589
1538a9e0 3590 radeon_ring_unlock_commit(rdev, ring, false);
48c0c902
AD		 3591
3592 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3593 ring = &rdev->ring[i];
3594 r = radeon_ring_lock(rdev, ring, 2);
71225e1c
NZ		 3595 if (r) {
3596 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3597 return r;
3598 }
48c0c902
AD		 3599
3600 /* clear the compute context state */
3601 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3602 radeon_ring_write(ring, 0);
3603
1538a9e0 3604 radeon_ring_unlock_commit(rdev, ring, false);
48c0c902
AD		 3605 }
3606
3607 return 0;
3608}
3609
3610static void si_cp_fini(struct radeon_device *rdev)
3611{
45df6803 3612 struct radeon_ring *ring;
48c0c902 3613 si_cp_enable(rdev, false);
45df6803
CK		 3614
3615 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3616 radeon_ring_fini(rdev, ring);
3617 radeon_scratch_free(rdev, ring->rptr_save_reg);
3618
3619 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3620 radeon_ring_fini(rdev, ring);
3621 radeon_scratch_free(rdev, ring->rptr_save_reg);
3622
3623 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3624 radeon_ring_fini(rdev, ring);
3625 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3626}
3627
3628static int si_cp_resume(struct radeon_device *rdev)
3629{
3630 struct radeon_ring *ring;
3631 u32 tmp;
3632 u32 rb_bufsz;
3633 int r;
3634
811e4d58
AD		 3635 si_enable_gui_idle_interrupt(rdev, false);
3636
48c0c902
AD		 3637 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3638 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3639
3640 /* Set the write pointer delay */
3641 WREG32(CP_RB_WPTR_DELAY, 0);
3642
3643 WREG32(CP_DEBUG, 0);
3644 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3645
3646 /* ring 0 - compute and gfx */
3647 /* Set ring buffer size */
3648 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3649 rb_bufsz = order_base_2(ring->ring_size / 8);
3650 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3651#ifdef __BIG_ENDIAN
3652 tmp |= BUF_SWAP_32BIT;
3653#endif
3654 WREG32(CP_RB0_CNTL, tmp);
3655
3656 /* Initialize the ring buffer's read and write pointers */
3657 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3658 ring->wptr = 0;
3659 WREG32(CP_RB0_WPTR, ring->wptr);
3660
48fc7f7e 3661 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3662 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3663 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3664
3665 if (rdev->wb.enabled)
3666 WREG32(SCRATCH_UMSK, 0xff);
3667 else {
3668 tmp |= RB_NO_UPDATE;
3669 WREG32(SCRATCH_UMSK, 0);
3670 }
3671
3672 mdelay(1);
3673 WREG32(CP_RB0_CNTL, tmp);
3674
3675 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3676
48c0c902
AD		 3677 /* ring1 - compute only */
3678 /* Set ring buffer size */
3679 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
b72a8925
DV		 3680 rb_bufsz = order_base_2(ring->ring_size / 8);
3681 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3682#ifdef __BIG_ENDIAN
3683 tmp |= BUF_SWAP_32BIT;
3684#endif
3685 WREG32(CP_RB1_CNTL, tmp);
3686
3687 /* Initialize the ring buffer's read and write pointers */
3688 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3689 ring->wptr = 0;
3690 WREG32(CP_RB1_WPTR, ring->wptr);
3691
48fc7f7e 3692 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3693 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3694 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3695
3696 mdelay(1);
3697 WREG32(CP_RB1_CNTL, tmp);
3698
3699 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3700
48c0c902
AD		 3701 /* ring2 - compute only */
3702 /* Set ring buffer size */
3703 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
b72a8925
DV		 3704 rb_bufsz = order_base_2(ring->ring_size / 8);
3705 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3706#ifdef __BIG_ENDIAN
3707 tmp |= BUF_SWAP_32BIT;
3708#endif
3709 WREG32(CP_RB2_CNTL, tmp);
3710
3711 /* Initialize the ring buffer's read and write pointers */
3712 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3713 ring->wptr = 0;
3714 WREG32(CP_RB2_WPTR, ring->wptr);
3715
48fc7f7e 3716 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3717 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3718 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3719
3720 mdelay(1);
3721 WREG32(CP_RB2_CNTL, tmp);
3722
3723 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3724
48c0c902
AD		 3725 /* start the rings */
3726 si_cp_start(rdev);
3727 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3728 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3729 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3730 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3731 if (r) {
3732 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3733 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3734 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3735 return r;
3736 }
3737 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3738 if (r) {
3739 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3740 }
3741 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3742 if (r) {
3743 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3744 }
3745
811e4d58
AD		 3746 si_enable_gui_idle_interrupt(rdev, true);
3747
50efa51a
AD		 3748 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3749 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3750
48c0c902
AD		 3751 return 0;
3752}
3753
2483b4ea 3754u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3755{
014bb209 3756 u32 reset_mask = 0;
1c534671 3757 u32 tmp;
06bc6df0 3758
014bb209
AD		 3759 /* GRBM_STATUS */
3760 tmp = RREG32(GRBM_STATUS);
3761 if (tmp & (PA_BUSY | SC_BUSY |
3762 BCI_BUSY | SX_BUSY |
3763 TA_BUSY | VGT_BUSY |
3764 DB_BUSY | CB_BUSY |
3765 GDS_BUSY | SPI_BUSY |
3766 IA_BUSY | IA_BUSY_NO_DMA))
3767 reset_mask |= RADEON_RESET_GFX;
3768
3769 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3770 CP_BUSY | CP_COHERENCY_BUSY))
3771 reset_mask |= RADEON_RESET_CP;
3772
3773 if (tmp & GRBM_EE_BUSY)
3774 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3775
3776 /* GRBM_STATUS2 */
3777 tmp = RREG32(GRBM_STATUS2);
3778 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3779 reset_mask |= RADEON_RESET_RLC;
3780
3781 /* DMA_STATUS_REG 0 */
3782 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3783 if (!(tmp & DMA_IDLE))
3784 reset_mask |= RADEON_RESET_DMA;
3785
3786 /* DMA_STATUS_REG 1 */
3787 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3788 if (!(tmp & DMA_IDLE))
3789 reset_mask |= RADEON_RESET_DMA1;
3790
3791 /* SRBM_STATUS2 */
3792 tmp = RREG32(SRBM_STATUS2);
3793 if (tmp & DMA_BUSY)
3794 reset_mask |= RADEON_RESET_DMA;
3795
3796 if (tmp & DMA1_BUSY)
3797 reset_mask |= RADEON_RESET_DMA1;
3798
3799 /* SRBM_STATUS */
3800 tmp = RREG32(SRBM_STATUS);
3801
3802 if (tmp & IH_BUSY)
3803 reset_mask |= RADEON_RESET_IH;
3804
3805 if (tmp & SEM_BUSY)
3806 reset_mask |= RADEON_RESET_SEM;
3807
3808 if (tmp & GRBM_RQ_PENDING)
3809 reset_mask |= RADEON_RESET_GRBM;
3810
3811 if (tmp & VMC_BUSY)
3812 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3813
014bb209
AD		 3814 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3815 MCC_BUSY | MCD_BUSY))
3816 reset_mask |= RADEON_RESET_MC;
3817
3818 if (evergreen_is_display_hung(rdev))
3819 reset_mask |= RADEON_RESET_DISPLAY;
3820
3821 /* VM_L2_STATUS */
3822 tmp = RREG32(VM_L2_STATUS);
3823 if (tmp & L2_BUSY)
3824 reset_mask |= RADEON_RESET_VMC;
3825
d808fc88
AD		 3826 /* Skip MC reset as it's mostly likely not hung, just busy */
3827 if (reset_mask & RADEON_RESET_MC) {
3828 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3829 reset_mask &= ~RADEON_RESET_MC;
3830 }
3831
014bb209
AD		 3832 return reset_mask;
3833}
3834
3835static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3836{
3837 struct evergreen_mc_save save;
3838 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3839 u32 tmp;
19fc42ed 3840
06bc6df0 3841 if (reset_mask == 0)
014bb209 3842 return;
06bc6df0
AD		 3843
3844 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3845
1c534671 3846 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3847 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3848 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3849 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3850 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3851
a6f4ae8d
AD		 3852 /* disable PG/CG */
3853 si_fini_pg(rdev);
3854 si_fini_cg(rdev);
3855
3856 /* stop the rlc */
3857 si_rlc_stop(rdev);
3858
1c534671
AD		 3859 /* Disable CP parsing/prefetching */
3860 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3861
3862 if (reset_mask & RADEON_RESET_DMA) {
3863 /* dma0 */
3864 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3865 tmp &= ~DMA_RB_ENABLE;
3866 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3867 }
3868 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3869 /* dma1 */
3870 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3871 tmp &= ~DMA_RB_ENABLE;
3872 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3873 }
3874
f770d78a
AD		 3875 udelay(50);
3876
3877 evergreen_mc_stop(rdev, &save);
3878 if (evergreen_mc_wait_for_idle(rdev)) {
3879 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3880 }
3881
1c534671
AD		 3882 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3883 grbm_soft_reset = SOFT_RESET_CB |
3884 SOFT_RESET_DB |
3885 SOFT_RESET_GDS |
3886 SOFT_RESET_PA |
3887 SOFT_RESET_SC |
3888 SOFT_RESET_BCI |
3889 SOFT_RESET_SPI |
3890 SOFT_RESET_SX |
3891 SOFT_RESET_TC |
3892 SOFT_RESET_TA |
3893 SOFT_RESET_VGT |
3894 SOFT_RESET_IA;
3895 }
3896
3897 if (reset_mask & RADEON_RESET_CP) {
3898 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3899
3900 srbm_soft_reset |= SOFT_RESET_GRBM;
3901 }
06bc6df0
AD		 3902
3903 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3904 srbm_soft_reset |= SOFT_RESET_DMA;
3905
3906 if (reset_mask & RADEON_RESET_DMA1)
3907 srbm_soft_reset |= SOFT_RESET_DMA1;
3908
3909 if (reset_mask & RADEON_RESET_DISPLAY)
3910 srbm_soft_reset |= SOFT_RESET_DC;
3911
3912 if (reset_mask & RADEON_RESET_RLC)
3913 grbm_soft_reset |= SOFT_RESET_RLC;
3914
3915 if (reset_mask & RADEON_RESET_SEM)
3916 srbm_soft_reset |= SOFT_RESET_SEM;
3917
3918 if (reset_mask & RADEON_RESET_IH)
3919 srbm_soft_reset |= SOFT_RESET_IH;
3920
3921 if (reset_mask & RADEON_RESET_GRBM)
3922 srbm_soft_reset |= SOFT_RESET_GRBM;
3923
3924 if (reset_mask & RADEON_RESET_VMC)
3925 srbm_soft_reset |= SOFT_RESET_VMC;
3926
3927 if (reset_mask & RADEON_RESET_MC)
3928 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3929
3930 if (grbm_soft_reset) {
3931 tmp = RREG32(GRBM_SOFT_RESET);
3932 tmp |= grbm_soft_reset;
3933 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3934 WREG32(GRBM_SOFT_RESET, tmp);
3935 tmp = RREG32(GRBM_SOFT_RESET);
3936
3937 udelay(50);
3938
3939 tmp &= ~grbm_soft_reset;
3940 WREG32(GRBM_SOFT_RESET, tmp);
3941 tmp = RREG32(GRBM_SOFT_RESET);
3942 }
3943
3944 if (srbm_soft_reset) {
3945 tmp = RREG32(SRBM_SOFT_RESET);
3946 tmp |= srbm_soft_reset;
3947 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3948 WREG32(SRBM_SOFT_RESET, tmp);
3949 tmp = RREG32(SRBM_SOFT_RESET);
3950
3951 udelay(50);
3952
3953 tmp &= ~srbm_soft_reset;
3954 WREG32(SRBM_SOFT_RESET, tmp);
3955 tmp = RREG32(SRBM_SOFT_RESET);
3956 }
06bc6df0
AD		 3957
3958 /* Wait a little for things to settle down */
3959 udelay(50);
3960
c476dde2 3961 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3962 udelay(50);
3963
1c534671 3964 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3965}
3966
4a5c8ea5
AD		 3967static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3968{
3969 u32 tmp, i;
3970
3971 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3972 tmp |= SPLL_BYPASS_EN;
3973 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3974
3975 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3976 tmp |= SPLL_CTLREQ_CHG;
3977 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3978
3979 for (i = 0; i < rdev->usec_timeout; i++) {
3980 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3981 break;
3982 udelay(1);
3983 }
3984
3985 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3986 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3987 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3988
3989 tmp = RREG32(MPLL_CNTL_MODE);
3990 tmp &= ~MPLL_MCLK_SEL;
3991 WREG32(MPLL_CNTL_MODE, tmp);
3992}
3993
3994static void si_spll_powerdown(struct radeon_device *rdev)
3995{
3996 u32 tmp;
3997
3998 tmp = RREG32(SPLL_CNTL_MODE);
3999 tmp |= SPLL_SW_DIR_CONTROL;
4000 WREG32(SPLL_CNTL_MODE, tmp);
4001
4002 tmp = RREG32(CG_SPLL_FUNC_CNTL);
4003 tmp |= SPLL_RESET;
4004 WREG32(CG_SPLL_FUNC_CNTL, tmp);
4005
4006 tmp = RREG32(CG_SPLL_FUNC_CNTL);
4007 tmp |= SPLL_SLEEP;
4008 WREG32(CG_SPLL_FUNC_CNTL, tmp);
4009
4010 tmp = RREG32(SPLL_CNTL_MODE);
4011 tmp &= ~SPLL_SW_DIR_CONTROL;
4012 WREG32(SPLL_CNTL_MODE, tmp);
4013}
4014
4015static void si_gpu_pci_config_reset(struct radeon_device *rdev)
4016{
4017 struct evergreen_mc_save save;
4018 u32 tmp, i;
4019
4020 dev_info(rdev->dev, "GPU pci config reset\n");
4021
4022 /* disable dpm? */
4023
4024 /* disable cg/pg */
4025 si_fini_pg(rdev);
4026 si_fini_cg(rdev);
4027
4028 /* Disable CP parsing/prefetching */
4029 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4030 /* dma0 */
4031 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4032 tmp &= ~DMA_RB_ENABLE;
4033 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4034 /* dma1 */
4035 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4036 tmp &= ~DMA_RB_ENABLE;
4037 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4038 /* XXX other engines? */
4039
4040 /* halt the rlc, disable cp internal ints */
4041 si_rlc_stop(rdev);
4042
4043 udelay(50);
4044
4045 /* disable mem access */
4046 evergreen_mc_stop(rdev, &save);
4047 if (evergreen_mc_wait_for_idle(rdev)) {
4048 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4049 }
4050
4051 /* set mclk/sclk to bypass */
4052 si_set_clk_bypass_mode(rdev);
4053 /* powerdown spll */
4054 si_spll_powerdown(rdev);
4055 /* disable BM */
4056 pci_clear_master(rdev->pdev);
4057 /* reset */
4058 radeon_pci_config_reset(rdev);
4059 /* wait for asic to come out of reset */
4060 for (i = 0; i < rdev->usec_timeout; i++) {
4061 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4062 break;
4063 udelay(1);
4064 }
4065}
4066
71fe2899 4067int si_asic_reset(struct radeon_device *rdev, bool hard)
c476dde2 4068{
014bb209
AD		 4069 u32 reset_mask;
4070
71fe2899
JG		 4071 if (hard) {
4072 si_gpu_pci_config_reset(rdev);
4073 return 0;
4074 }
4075
014bb209
AD		 4076 reset_mask = si_gpu_check_soft_reset(rdev);
4077
4078 if (reset_mask)
4079 r600_set_bios_scratch_engine_hung(rdev, true);
4080
4a5c8ea5 4081 /* try soft reset */
014bb209
AD		 4082 si_gpu_soft_reset(rdev, reset_mask);
4083
4084 reset_mask = si_gpu_check_soft_reset(rdev);
4085
4a5c8ea5
AD		 4086 /* try pci config reset */
4087 if (reset_mask && radeon_hard_reset)
4088 si_gpu_pci_config_reset(rdev);
4089
4090 reset_mask = si_gpu_check_soft_reset(rdev);
4091
014bb209
AD		 4092 if (!reset_mask)
4093 r600_set_bios_scratch_engine_hung(rdev, false);
4094
4095 return 0;
c476dde2
AD		 4096}
4097
123bc183
AD		 4098/**
4099 * si_gfx_is_lockup - Check if the GFX engine is locked up
4100 *
4101 * @rdev: radeon_device pointer
4102 * @ring: radeon_ring structure holding ring information
4103 *
4104 * Check if the GFX engine is locked up.
4105 * Returns true if the engine appears to be locked up, false if not.
4106 */
4107bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4108{
4109 u32 reset_mask = si_gpu_check_soft_reset(rdev);
4110
4111 if (!(reset_mask & (RADEON_RESET_GFX |
4112 RADEON_RESET_COMPUTE |
4113 RADEON_RESET_CP))) {
ff212f25 4114 radeon_ring_lockup_update(rdev, ring);
123bc183
AD		 4115 return false;
4116 }
123bc183
AD		 4117 return radeon_ring_test_lockup(rdev, ring);
4118}
4119
d2800ee5
AD		 4120/* MC */
4121static void si_mc_program(struct radeon_device *rdev)
4122{
4123 struct evergreen_mc_save save;
4124 u32 tmp;
4125 int i, j;
4126
4127 /* Initialize HDP */
4128 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4129 WREG32((0x2c14 + j), 0x00000000);
4130 WREG32((0x2c18 + j), 0x00000000);
4131 WREG32((0x2c1c + j), 0x00000000);
4132 WREG32((0x2c20 + j), 0x00000000);
4133 WREG32((0x2c24 + j), 0x00000000);
4134 }
4135 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4136
4137 evergreen_mc_stop(rdev, &save);
4138 if (radeon_mc_wait_for_idle(rdev)) {
4139 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4140 }
5153550a
AD		 4141 if (!ASIC_IS_NODCE(rdev))
4142 /* Lockout access through VGA aperture*/
4143 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 4144 /* Update configuration */
4145 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4146 rdev->mc.vram_start >> 12);
4147 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4148 rdev->mc.vram_end >> 12);
4149 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4150 rdev->vram_scratch.gpu_addr >> 12);
4151 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4152 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4153 WREG32(MC_VM_FB_LOCATION, tmp);
4154 /* XXX double check these! */
4155 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4156 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4157 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4158 WREG32(MC_VM_AGP_BASE, 0);
4159 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4160 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4161 if (radeon_mc_wait_for_idle(rdev)) {
4162 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4163 }
4164 evergreen_mc_resume(rdev, &save);
5153550a
AD		 4165 if (!ASIC_IS_NODCE(rdev)) {
4166 /* we need to own VRAM, so turn off the VGA renderer here
4167 * to stop it overwriting our objects */
4168 rv515_vga_render_disable(rdev);
4169 }
d2800ee5
AD		 4170}
4171
1c49165d
AD		 4172void si_vram_gtt_location(struct radeon_device *rdev,
4173 struct radeon_mc *mc)
d2800ee5
AD		 4174{
4175 if (mc->mc_vram_size > 0xFFC0000000ULL) {
4176 /* leave room for at least 1024M GTT */
4177 dev_warn(rdev->dev, "limiting VRAM\n");
4178 mc->real_vram_size = 0xFFC0000000ULL;
4179 mc->mc_vram_size = 0xFFC0000000ULL;
4180 }
9ed8b1f9 4181 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 4182 rdev->mc.gtt_base_align = 0;
9ed8b1f9 4183 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 4184}
4185
4186static int si_mc_init(struct radeon_device *rdev)
4187{
4188 u32 tmp;
4189 int chansize, numchan;
4190
4191 /* Get VRAM informations */
4192 rdev->mc.vram_is_ddr = true;
4193 tmp = RREG32(MC_ARB_RAMCFG);
4194 if (tmp & CHANSIZE_OVERRIDE) {
4195 chansize = 16;
4196 } else if (tmp & CHANSIZE_MASK) {
4197 chansize = 64;
4198 } else {
4199 chansize = 32;
4200 }
4201 tmp = RREG32(MC_SHARED_CHMAP);
4202 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4203 case 0:
4204 default:
4205 numchan = 1;
4206 break;
4207 case 1:
4208 numchan = 2;
4209 break;
4210 case 2:
4211 numchan = 4;
4212 break;
4213 case 3:
4214 numchan = 8;
4215 break;
4216 case 4:
4217 numchan = 3;
4218 break;
4219 case 5:
4220 numchan = 6;
4221 break;
4222 case 6:
4223 numchan = 10;
4224 break;
4225 case 7:
4226 numchan = 12;
4227 break;
4228 case 8:
4229 numchan = 16;
4230 break;
4231 }
4232 rdev->mc.vram_width = numchan * chansize;
4233 /* Could aper size report 0 ? */
4234 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4235 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4236 /* size in MB on si */
0ca223b0
AD		 4237 tmp = RREG32(CONFIG_MEMSIZE);
4238 /* some boards may have garbage in the upper 16 bits */
4239 if (tmp & 0xffff0000) {
4240 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4241 if (tmp & 0xffff)
4242 tmp &= 0xffff;
4243 }
4244 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4245 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
d2800ee5
AD		 4246 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4247 si_vram_gtt_location(rdev, &rdev->mc);
4248 radeon_update_bandwidth_info(rdev);
4249
4250 return 0;
4251}
4252
4253/*
4254 * GART
4255 */
4256void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4257{
4258 /* flush hdp cache */
4259 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4260
4261 /* bits 0-15 are the VM contexts0-15 */
4262 WREG32(VM_INVALIDATE_REQUEST, 1);
4263}
4264
1109ca09 4265static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 4266{
4267 int r, i;
4268
4269 if (rdev->gart.robj == NULL) {
4270 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4271 return -EINVAL;
4272 }
4273 r = radeon_gart_table_vram_pin(rdev);
4274 if (r)
4275 return r;
d2800ee5
AD		 4276 /* Setup TLB control */
4277 WREG32(MC_VM_MX_L1_TLB_CNTL,
4278 (0xA << 7) |
4279 ENABLE_L1_TLB |
ec3dbbcb 4280 ENABLE_L1_FRAGMENT_PROCESSING |
d2800ee5
AD		 4281 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4282 ENABLE_ADVANCED_DRIVER_MODEL |
4283 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4284 /* Setup L2 cache */
4285 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
ec3dbbcb 4286 ENABLE_L2_FRAGMENT_PROCESSING |
d2800ee5
AD		 4287 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4288 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4289 EFFECTIVE_L2_QUEUE_SIZE(7) |
4290 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4291 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4292 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 4293 BANK_SELECT(4) |
4294 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
d2800ee5
AD		 4295 /* setup context0 */
4296 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
7c0411d2 4297 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
d2800ee5
AD		 4298 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4299 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4300 (u32)(rdev->dummy_page.addr >> 12));
4301 WREG32(VM_CONTEXT0_CNTL2, 0);
4302 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4303 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4304
4305 WREG32(0x15D4, 0);
4306 WREG32(0x15D8, 0);
4307 WREG32(0x15DC, 0);
4308
4309 /* empty context1-15 */
d2800ee5
AD		 4310 /* set vm size, must be a multiple of 4 */
4311 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
607d4806 4312 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
23d4f1f2
AD		 4313 /* Assign the pt base to something valid for now; the pts used for
4314 * the VMs are determined by the application and setup and assigned
4315 * on the fly in the vm part of radeon_gart.c
4316 */
d2800ee5
AD		 4317 for (i = 1; i < 16; i++) {
4318 if (i < 8)
4319 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
054e01d6 4320 rdev->vm_manager.saved_table_addr[i]);
d2800ee5
AD		 4321 else
4322 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
054e01d6 4323 rdev->vm_manager.saved_table_addr[i]);
d2800ee5
AD		 4324 }
4325
4326 /* enable context1-15 */
4327 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4328 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 4329 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 4330 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 4331 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
ae133a11
CK		 4332 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4333 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4334 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4335 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4336 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4337 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4338 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4339 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4340 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4341 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4342 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4343 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 4344
4345 si_pcie_gart_tlb_flush(rdev);
4346 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4347 (unsigned)(rdev->mc.gtt_size >> 20),
4348 (unsigned long long)rdev->gart.table_addr);
4349 rdev->gart.ready = true;
4350 return 0;
4351}
4352
1109ca09 4353static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5 4354{
054e01d6
CK		 4355 unsigned i;
4356
4357 for (i = 1; i < 16; ++i) {
4358 uint32_t reg;
4359 if (i < 8)
4360 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4361 else
4362 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4363 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4364 }
4365
d2800ee5
AD		 4366 /* Disable all tables */
4367 WREG32(VM_CONTEXT0_CNTL, 0);
4368 WREG32(VM_CONTEXT1_CNTL, 0);
4369 /* Setup TLB control */
4370 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4371 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4372 /* Setup L2 cache */
4373 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4374 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4375 EFFECTIVE_L2_QUEUE_SIZE(7) |
4376 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4377 WREG32(VM_L2_CNTL2, 0);
4378 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4379 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4380 radeon_gart_table_vram_unpin(rdev);
4381}
4382
1109ca09 4383static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 4384{
4385 si_pcie_gart_disable(rdev);
4386 radeon_gart_table_vram_free(rdev);
4387 radeon_gart_fini(rdev);
4388}
4389
498dd8b3
AD		 4390/* vm parser */
4391static bool si_vm_reg_valid(u32 reg)
4392{
4393 /* context regs are fine */
4394 if (reg >= 0x28000)
4395 return true;
4396
3d02b7fe
BN		 4397 /* shader regs are also fine */
4398 if (reg >= 0xB000 && reg < 0xC000)
4399 return true;
4400
498dd8b3
AD		 4401 /* check config regs */
4402 switch (reg) {
4403 case GRBM_GFX_INDEX:
f418b88a 4404 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4405 case VGT_VTX_VECT_EJECT_REG:
4406 case VGT_CACHE_INVALIDATION:
4407 case VGT_ESGS_RING_SIZE:
4408 case VGT_GSVS_RING_SIZE:
4409 case VGT_GS_VERTEX_REUSE:
4410 case VGT_PRIMITIVE_TYPE:
4411 case VGT_INDEX_TYPE:
4412 case VGT_NUM_INDICES:
4413 case VGT_NUM_INSTANCES:
4414 case VGT_TF_RING_SIZE:
4415 case VGT_HS_OFFCHIP_PARAM:
4416 case VGT_TF_MEMORY_BASE:
4417 case PA_CL_ENHANCE:
4418 case PA_SU_LINE_STIPPLE_VALUE:
4419 case PA_SC_LINE_STIPPLE_STATE:
4420 case PA_SC_ENHANCE:
4421 case SQC_CACHES:
4422 case SPI_STATIC_THREAD_MGMT_1:
4423 case SPI_STATIC_THREAD_MGMT_2:
4424 case SPI_STATIC_THREAD_MGMT_3:
4425 case SPI_PS_MAX_WAVE_ID:
4426 case SPI_CONFIG_CNTL:
4427 case SPI_CONFIG_CNTL_1:
4428 case TA_CNTL_AUX:
113d0f9d 4429 case TA_CS_BC_BASE_ADDR:
498dd8b3
AD		 4430 return true;
4431 default:
4432 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4433 return false;
4434 }
4435}
4436
4437static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4438 u32 *ib, struct radeon_cs_packet *pkt)
4439{
4440 switch (pkt->opcode) {
4441 case PACKET3_NOP:
4442 case PACKET3_SET_BASE:
4443 case PACKET3_SET_CE_DE_COUNTERS:
4444 case PACKET3_LOAD_CONST_RAM:
4445 case PACKET3_WRITE_CONST_RAM:
4446 case PACKET3_WRITE_CONST_RAM_OFFSET:
4447 case PACKET3_DUMP_CONST_RAM:
4448 case PACKET3_INCREMENT_CE_COUNTER:
4449 case PACKET3_WAIT_ON_DE_COUNTER:
4450 case PACKET3_CE_WRITE:
4451 break;
4452 default:
4453 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4454 return -EINVAL;
4455 }
4456 return 0;
4457}
4458
e5b9e750
TS		 4459static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4460{
4461 u32 start_reg, reg, i;
4462 u32 command = ib[idx + 4];
4463 u32 info = ib[idx + 1];
4464 u32 idx_value = ib[idx];
4465 if (command & PACKET3_CP_DMA_CMD_SAS) {
4466 /* src address space is register */
4467 if (((info & 0x60000000) >> 29) == 0) {
4468 start_reg = idx_value << 2;
4469 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4470 reg = start_reg;
4471 if (!si_vm_reg_valid(reg)) {
4472 DRM_ERROR("CP DMA Bad SRC register\n");
4473 return -EINVAL;
4474 }
4475 } else {
4476 for (i = 0; i < (command & 0x1fffff); i++) {
4477 reg = start_reg + (4 * i);
4478 if (!si_vm_reg_valid(reg)) {
4479 DRM_ERROR("CP DMA Bad SRC register\n");
4480 return -EINVAL;
4481 }
4482 }
4483 }
4484 }
4485 }
4486 if (command & PACKET3_CP_DMA_CMD_DAS) {
4487 /* dst address space is register */
4488 if (((info & 0x00300000) >> 20) == 0) {
4489 start_reg = ib[idx + 2];
4490 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4491 reg = start_reg;
4492 if (!si_vm_reg_valid(reg)) {
4493 DRM_ERROR("CP DMA Bad DST register\n");
4494 return -EINVAL;
4495 }
4496 } else {
4497 for (i = 0; i < (command & 0x1fffff); i++) {
4498 reg = start_reg + (4 * i);
611ed9a5 4499 if (!si_vm_reg_valid(reg)) {
e5b9e750
TS		 4500 DRM_ERROR("CP DMA Bad DST register\n");
4501 return -EINVAL;
4502 }
4503 }
4504 }
4505 }
4506 }
4507 return 0;
4508}
4509
498dd8b3
AD		 4510static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4511 u32 *ib, struct radeon_cs_packet *pkt)
4512{
e5b9e750 4513 int r;
498dd8b3
AD		 4514 u32 idx = pkt->idx + 1;
4515 u32 idx_value = ib[idx];
4516 u32 start_reg, end_reg, reg, i;
4517
4518 switch (pkt->opcode) {
4519 case PACKET3_NOP:
4520 case PACKET3_SET_BASE:
4521 case PACKET3_CLEAR_STATE:
4522 case PACKET3_INDEX_BUFFER_SIZE:
4523 case PACKET3_DISPATCH_DIRECT:
4524 case PACKET3_DISPATCH_INDIRECT:
4525 case PACKET3_ALLOC_GDS:
4526 case PACKET3_WRITE_GDS_RAM:
4527 case PACKET3_ATOMIC_GDS:
4528 case PACKET3_ATOMIC:
4529 case PACKET3_OCCLUSION_QUERY:
4530 case PACKET3_SET_PREDICATION:
4531 case PACKET3_COND_EXEC:
4532 case PACKET3_PRED_EXEC:
4533 case PACKET3_DRAW_INDIRECT:
4534 case PACKET3_DRAW_INDEX_INDIRECT:
4535 case PACKET3_INDEX_BASE:
4536 case PACKET3_DRAW_INDEX_2:
4537 case PACKET3_CONTEXT_CONTROL:
4538 case PACKET3_INDEX_TYPE:
4539 case PACKET3_DRAW_INDIRECT_MULTI:
4540 case PACKET3_DRAW_INDEX_AUTO:
4541 case PACKET3_DRAW_INDEX_IMMD:
4542 case PACKET3_NUM_INSTANCES:
4543 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4544 case PACKET3_STRMOUT_BUFFER_UPDATE:
4545 case PACKET3_DRAW_INDEX_OFFSET_2:
4546 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4547 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4548 case PACKET3_MPEG_INDEX:
4549 case PACKET3_WAIT_REG_MEM:
4550 case PACKET3_MEM_WRITE:
4551 case PACKET3_PFP_SYNC_ME:
4552 case PACKET3_SURFACE_SYNC:
4553 case PACKET3_EVENT_WRITE:
4554 case PACKET3_EVENT_WRITE_EOP:
4555 case PACKET3_EVENT_WRITE_EOS:
4556 case PACKET3_SET_CONTEXT_REG:
4557 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4558 case PACKET3_SET_SH_REG:
4559 case PACKET3_SET_SH_REG_OFFSET:
4560 case PACKET3_INCREMENT_DE_COUNTER:
4561 case PACKET3_WAIT_ON_CE_COUNTER:
4562 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4563 case PACKET3_ME_WRITE:
4564 break;
4565 case PACKET3_COPY_DATA:
4566 if ((idx_value & 0xf00) == 0) {
4567 reg = ib[idx + 3] * 4;
4568 if (!si_vm_reg_valid(reg))
4569 return -EINVAL;
4570 }
4571 break;
4572 case PACKET3_WRITE_DATA:
4573 if ((idx_value & 0xf00) == 0) {
4574 start_reg = ib[idx + 1] * 4;
4575 if (idx_value & 0x10000) {
4576 if (!si_vm_reg_valid(start_reg))
4577 return -EINVAL;
4578 } else {
4579 for (i = 0; i < (pkt->count - 2); i++) {
4580 reg = start_reg + (4 * i);
4581 if (!si_vm_reg_valid(reg))
4582 return -EINVAL;
4583 }
4584 }
4585 }
4586 break;
4587 case PACKET3_COND_WRITE:
4588 if (idx_value & 0x100) {
4589 reg = ib[idx + 5] * 4;
4590 if (!si_vm_reg_valid(reg))
4591 return -EINVAL;
4592 }
4593 break;
4594 case PACKET3_COPY_DW:
4595 if (idx_value & 0x2) {
4596 reg = ib[idx + 3] * 4;
4597 if (!si_vm_reg_valid(reg))
4598 return -EINVAL;
4599 }
4600 break;
4601 case PACKET3_SET_CONFIG_REG:
4602 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4603 end_reg = 4 * pkt->count + start_reg - 4;
4604 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4605 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4606 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4607 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4608 return -EINVAL;
4609 }
4610 for (i = 0; i < pkt->count; i++) {
4611 reg = start_reg + (4 * i);
4612 if (!si_vm_reg_valid(reg))
4613 return -EINVAL;
4614 }
4615 break;
5aa709be 4616 case PACKET3_CP_DMA:
e5b9e750
TS		 4617 r = si_vm_packet3_cp_dma_check(ib, idx);
4618 if (r)
4619 return r;
5aa709be 4620 break;
498dd8b3
AD		 4621 default:
4622 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4623 return -EINVAL;
4624 }
4625 return 0;
4626}
4627
4628static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4629 u32 *ib, struct radeon_cs_packet *pkt)
4630{
e5b9e750 4631 int r;
498dd8b3
AD		 4632 u32 idx = pkt->idx + 1;
4633 u32 idx_value = ib[idx];
4634 u32 start_reg, reg, i;
4635
4636 switch (pkt->opcode) {
4637 case PACKET3_NOP:
4638 case PACKET3_SET_BASE:
4639 case PACKET3_CLEAR_STATE:
4640 case PACKET3_DISPATCH_DIRECT:
4641 case PACKET3_DISPATCH_INDIRECT:
4642 case PACKET3_ALLOC_GDS:
4643 case PACKET3_WRITE_GDS_RAM:
4644 case PACKET3_ATOMIC_GDS:
4645 case PACKET3_ATOMIC:
4646 case PACKET3_OCCLUSION_QUERY:
4647 case PACKET3_SET_PREDICATION:
4648 case PACKET3_COND_EXEC:
4649 case PACKET3_PRED_EXEC:
4650 case PACKET3_CONTEXT_CONTROL:
4651 case PACKET3_STRMOUT_BUFFER_UPDATE:
4652 case PACKET3_WAIT_REG_MEM:
4653 case PACKET3_MEM_WRITE:
4654 case PACKET3_PFP_SYNC_ME:
4655 case PACKET3_SURFACE_SYNC:
4656 case PACKET3_EVENT_WRITE:
4657 case PACKET3_EVENT_WRITE_EOP:
4658 case PACKET3_EVENT_WRITE_EOS:
4659 case PACKET3_SET_CONTEXT_REG:
4660 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4661 case PACKET3_SET_SH_REG:
4662 case PACKET3_SET_SH_REG_OFFSET:
4663 case PACKET3_INCREMENT_DE_COUNTER:
4664 case PACKET3_WAIT_ON_CE_COUNTER:
4665 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4666 case PACKET3_ME_WRITE:
4667 break;
4668 case PACKET3_COPY_DATA:
4669 if ((idx_value & 0xf00) == 0) {
4670 reg = ib[idx + 3] * 4;
4671 if (!si_vm_reg_valid(reg))
4672 return -EINVAL;
4673 }
4674 break;
4675 case PACKET3_WRITE_DATA:
4676 if ((idx_value & 0xf00) == 0) {
4677 start_reg = ib[idx + 1] * 4;
4678 if (idx_value & 0x10000) {
4679 if (!si_vm_reg_valid(start_reg))
4680 return -EINVAL;
4681 } else {
4682 for (i = 0; i < (pkt->count - 2); i++) {
4683 reg = start_reg + (4 * i);
4684 if (!si_vm_reg_valid(reg))
4685 return -EINVAL;
4686 }
4687 }
4688 }
4689 break;
4690 case PACKET3_COND_WRITE:
4691 if (idx_value & 0x100) {
4692 reg = ib[idx + 5] * 4;
4693 if (!si_vm_reg_valid(reg))
4694 return -EINVAL;
4695 }
4696 break;
4697 case PACKET3_COPY_DW:
4698 if (idx_value & 0x2) {
4699 reg = ib[idx + 3] * 4;
4700 if (!si_vm_reg_valid(reg))
4701 return -EINVAL;
4702 }
4703 break;
e5b9e750
TS		 4704 case PACKET3_CP_DMA:
4705 r = si_vm_packet3_cp_dma_check(ib, idx);
4706 if (r)
4707 return r;
4708 break;
498dd8b3
AD		 4709 default:
4710 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4711 return -EINVAL;
4712 }
4713 return 0;
4714}
4715
4716int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4717{
4718 int ret = 0;
9d0223d5 4719 u32 idx = 0, i;
498dd8b3
AD		 4720 struct radeon_cs_packet pkt;
4721
4722 do {
4723 pkt.idx = idx;
4e872ae2
IH		 4724 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4725 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4726 pkt.one_reg_wr = 0;
4727 switch (pkt.type) {
4e872ae2 4728 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4729 dev_err(rdev->dev, "Packet0 not allowed!\n");
4730 ret = -EINVAL;
4731 break;
4e872ae2 4732 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4733 idx += 1;
4734 break;
4e872ae2
IH		 4735 case RADEON_PACKET_TYPE3:
4736 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4737 if (ib->is_const_ib)
4738 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4739 else {
876dc9f3 4740 switch (ib->ring) {
498dd8b3
AD		 4741 case RADEON_RING_TYPE_GFX_INDEX:
4742 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4743 break;
4744 case CAYMAN_RING_TYPE_CP1_INDEX:
4745 case CAYMAN_RING_TYPE_CP2_INDEX:
4746 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4747 break;
4748 default:
876dc9f3 4749 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4750 ret = -EINVAL;
4751 break;
4752 }
4753 }
4754 idx += pkt.count + 2;
4755 break;
4756 default:
4757 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4758 ret = -EINVAL;
4759 break;
4760 }
e1b4e722
AD		 4761 if (ret) {
4762 for (i = 0; i < ib->length_dw; i++) {
4763 if (i == idx)
4764 printk("\t0x%08x <---\n", ib->ptr[i]);
4765 else
4766 printk("\t0x%08x\n", ib->ptr[i]);
4767 }
498dd8b3 4768 break;
e1b4e722 4769 }
498dd8b3
AD		 4770 } while (idx < ib->length_dw);
4771
4772 return ret;
4773}
4774
d2800ee5
AD		 4775/*
4776 * vm
4777 */
4778int si_vm_init(struct radeon_device *rdev)
4779{
4780 /* number of VMs */
4781 rdev->vm_manager.nvm = 16;
4782 /* base offset of vram pages */
4783 rdev->vm_manager.vram_base_offset = 0;
4784
4785 return 0;
4786}
4787
4788void si_vm_fini(struct radeon_device *rdev)
4789{
4790}
4791
fbf6dc7a
AD		 4792/**
4793 * si_vm_decode_fault - print human readable fault info
4794 *
4795 * @rdev: radeon_device pointer
4796 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4797 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4798 *
4799 * Print human readable fault information (SI).
4800 */
4801static void si_vm_decode_fault(struct radeon_device *rdev,
4802 u32 status, u32 addr)
4803{
4804 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4805 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4806 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4807 char *block;
4808
4809 if (rdev->family == CHIP_TAHITI) {
4810 switch (mc_id) {
4811 case 160:
4812 case 144:
4813 case 96:
4814 case 80:
4815 case 224:
4816 case 208:
4817 case 32:
4818 case 16:
4819 block = "CB";
4820 break;
4821 case 161:
4822 case 145:
4823 case 97:
4824 case 81:
4825 case 225:
4826 case 209:
4827 case 33:
4828 case 17:
4829 block = "CB_FMASK";
4830 break;
4831 case 162:
4832 case 146:
4833 case 98:
4834 case 82:
4835 case 226:
4836 case 210:
4837 case 34:
4838 case 18:
4839 block = "CB_CMASK";
4840 break;
4841 case 163:
4842 case 147:
4843 case 99:
4844 case 83:
4845 case 227:
4846 case 211:
4847 case 35:
4848 case 19:
4849 block = "CB_IMMED";
4850 break;
4851 case 164:
4852 case 148:
4853 case 100:
4854 case 84:
4855 case 228:
4856 case 212:
4857 case 36:
4858 case 20:
4859 block = "DB";
4860 break;
4861 case 165:
4862 case 149:
4863 case 101:
4864 case 85:
4865 case 229:
4866 case 213:
4867 case 37:
4868 case 21:
4869 block = "DB_HTILE";
4870 break;
4871 case 167:
4872 case 151:
4873 case 103:
4874 case 87:
4875 case 231:
4876 case 215:
4877 case 39:
4878 case 23:
4879 block = "DB_STEN";
4880 break;
4881 case 72:
4882 case 68:
4883 case 64:
4884 case 8:
4885 case 4:
4886 case 0:
4887 case 136:
4888 case 132:
4889 case 128:
4890 case 200:
4891 case 196:
4892 case 192:
4893 block = "TC";
4894 break;
4895 case 112:
4896 case 48:
4897 block = "CP";
4898 break;
4899 case 49:
4900 case 177:
4901 case 50:
4902 case 178:
4903 block = "SH";
4904 break;
4905 case 53:
4906 case 190:
4907 block = "VGT";
4908 break;
4909 case 117:
4910 block = "IH";
4911 break;
4912 case 51:
4913 case 115:
4914 block = "RLC";
4915 break;
4916 case 119:
4917 case 183:
4918 block = "DMA0";
4919 break;
4920 case 61:
4921 block = "DMA1";
4922 break;
4923 case 248:
4924 case 120:
4925 block = "HDP";
4926 break;
4927 default:
4928 block = "unknown";
4929 break;
4930 }
4931 } else {
4932 switch (mc_id) {
4933 case 32:
4934 case 16:
4935 case 96:
4936 case 80:
4937 case 160:
4938 case 144:
4939 case 224:
4940 case 208:
4941 block = "CB";
4942 break;
4943 case 33:
4944 case 17:
4945 case 97:
4946 case 81:
4947 case 161:
4948 case 145:
4949 case 225:
4950 case 209:
4951 block = "CB_FMASK";
4952 break;
4953 case 34:
4954 case 18:
4955 case 98:
4956 case 82:
4957 case 162:
4958 case 146:
4959 case 226:
4960 case 210:
4961 block = "CB_CMASK";
4962 break;
4963 case 35:
4964 case 19:
4965 case 99:
4966 case 83:
4967 case 163:
4968 case 147:
4969 case 227:
4970 case 211:
4971 block = "CB_IMMED";
4972 break;
4973 case 36:
4974 case 20:
4975 case 100:
4976 case 84:
4977 case 164:
4978 case 148:
4979 case 228:
4980 case 212:
4981 block = "DB";
4982 break;
4983 case 37:
4984 case 21:
4985 case 101:
4986 case 85:
4987 case 165:
4988 case 149:
4989 case 229:
4990 case 213:
4991 block = "DB_HTILE";
4992 break;
4993 case 39:
4994 case 23:
4995 case 103:
4996 case 87:
4997 case 167:
4998 case 151:
4999 case 231:
5000 case 215:
5001 block = "DB_STEN";
5002 break;
5003 case 72:
5004 case 68:
5005 case 8:
5006 case 4:
5007 case 136:
5008 case 132:
5009 case 200:
5010 case 196:
5011 block = "TC";
5012 break;
5013 case 112:
5014 case 48:
5015 block = "CP";
5016 break;
5017 case 49:
5018 case 177:
5019 case 50:
5020 case 178:
5021 block = "SH";
5022 break;
5023 case 53:
5024 block = "VGT";
5025 break;
5026 case 117:
5027 block = "IH";
5028 break;
5029 case 51:
5030 case 115:
5031 block = "RLC";
5032 break;
5033 case 119:
5034 case 183:
5035 block = "DMA0";
5036 break;
5037 case 61:
5038 block = "DMA1";
5039 break;
5040 case 248:
5041 case 120:
5042 block = "HDP";
5043 break;
5044 default:
5045 block = "unknown";
5046 break;
5047 }
5048 }
5049
5050 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5051 protections, vmid, addr,
5052 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5053 block, mc_id);
5054}
5055
faffaf62
CK		 5056void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5057 unsigned vm_id, uint64_t pd_addr)
d2800ee5 5058{
76c44f2c
AD		 5059 /* write new base address */
5060 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 5061 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
76c44f2c
AD		 5062 WRITE_DATA_DST_SEL(0)));
5063
faffaf62 5064 if (vm_id < 8) {
76c44f2c 5065 radeon_ring_write(ring,
faffaf62 5066 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
ee60e29f 5067 } else {
76c44f2c 5068 radeon_ring_write(ring,
faffaf62 5069 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
ee60e29f 5070 }
76c44f2c 5071 radeon_ring_write(ring, 0);
faffaf62 5072 radeon_ring_write(ring, pd_addr >> 12);
ee60e29f 5073
d2800ee5 5074 /* flush hdp cache */
76c44f2c 5075 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5076 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
76c44f2c
AD		 5077 WRITE_DATA_DST_SEL(0)));
5078 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5079 radeon_ring_write(ring, 0);
ee60e29f
CK		 5080 radeon_ring_write(ring, 0x1);
5081
d2800ee5 5082 /* bits 0-15 are the VM contexts0-15 */
76c44f2c 5083 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5084 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
76c44f2c
AD		 5085 WRITE_DATA_DST_SEL(0)));
5086 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5087 radeon_ring_write(ring, 0);
faffaf62 5088 radeon_ring_write(ring, 1 << vm_id);
58f8cf56 5089
d474ea7e
AD		 5090 /* wait for the invalidate to complete */
5091 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5092 radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) | /* always */
5093 WAIT_REG_MEM_ENGINE(0))); /* me */
5094 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5095 radeon_ring_write(ring, 0);
5096 radeon_ring_write(ring, 0); /* ref */
5097 radeon_ring_write(ring, 0); /* mask */
5098 radeon_ring_write(ring, 0x20); /* poll interval */
5099
58f8cf56
CK		 5100 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5101 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5102 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 5103}
5104
f8f84ac5
AD		 5105/*
5106 * Power and clock gating
5107 */
5108static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5109{
5110 int i;
5111
5112 for (i = 0; i < rdev->usec_timeout; i++) {
5113 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5114 break;
5115 udelay(1);
5116 }
5117
5118 for (i = 0; i < rdev->usec_timeout; i++) {
5119 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5120 break;
5121 udelay(1);
5122 }
5123}
5124
5125static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5126 bool enable)
5127{
5128 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5129 u32 mask;
5130 int i;
5131
5132 if (enable)
5133 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5134 else
5135 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5136 WREG32(CP_INT_CNTL_RING0, tmp);
5137
5138 if (!enable) {
5139 /* read a gfx register */
5140 tmp = RREG32(DB_DEPTH_INFO);
5141
5142 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5143 for (i = 0; i < rdev->usec_timeout; i++) {
5144 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5145 break;
5146 udelay(1);
5147 }
5148 }
5149}
5150
5151static void si_set_uvd_dcm(struct radeon_device *rdev,
5152 bool sw_mode)
5153{
5154 u32 tmp, tmp2;
5155
5156 tmp = RREG32(UVD_CGC_CTRL);
5157 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5158 tmp |= DCM | CG_DT(1) | CLK_OD(4);
5159
5160 if (sw_mode) {
5161 tmp &= ~0x7ffff800;
5162 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5163 } else {
5164 tmp |= 0x7ffff800;
5165 tmp2 = 0;
5166 }
5167
5168 WREG32(UVD_CGC_CTRL, tmp);
5169 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5170}
5171
22c775ce 5172void si_init_uvd_internal_cg(struct radeon_device *rdev)
f8f84ac5
AD		 5173{
5174 bool hw_mode = true;
5175
5176 if (hw_mode) {
5177 si_set_uvd_dcm(rdev, false);
5178 } else {
5179 u32 tmp = RREG32(UVD_CGC_CTRL);
5180 tmp &= ~DCM;
5181 WREG32(UVD_CGC_CTRL, tmp);
5182 }
5183}
5184
5185static u32 si_halt_rlc(struct radeon_device *rdev)
5186{
5187 u32 data, orig;
5188
5189 orig = data = RREG32(RLC_CNTL);
5190
5191 if (data & RLC_ENABLE) {
5192 data &= ~RLC_ENABLE;
5193 WREG32(RLC_CNTL, data);
5194
5195 si_wait_for_rlc_serdes(rdev);
5196 }
5197
5198 return orig;
5199}
5200
5201static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5202{
5203 u32 tmp;
5204
5205 tmp = RREG32(RLC_CNTL);
5206 if (tmp != rlc)
5207 WREG32(RLC_CNTL, rlc);
5208}
5209
5210static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5211{
5212 u32 data, orig;
5213
5214 orig = data = RREG32(DMA_PG);
e16866ec 5215 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
f8f84ac5
AD		 5216 data |= PG_CNTL_ENABLE;
5217 else
5218 data &= ~PG_CNTL_ENABLE;
5219 if (orig != data)
5220 WREG32(DMA_PG, data);
5221}
5222
5223static void si_init_dma_pg(struct radeon_device *rdev)
5224{
5225 u32 tmp;
5226
5227 WREG32(DMA_PGFSM_WRITE, 0x00002000);
5228 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5229
5230 for (tmp = 0; tmp < 5; tmp++)
5231 WREG32(DMA_PGFSM_WRITE, 0);
5232}
5233
5234static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5235 bool enable)
5236{
5237 u32 tmp;
5238
2b19d17f 5239 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
f8f84ac5
AD		 5240 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5241 WREG32(RLC_TTOP_D, tmp);
5242
5243 tmp = RREG32(RLC_PG_CNTL);
5244 tmp |= GFX_PG_ENABLE;
5245 WREG32(RLC_PG_CNTL, tmp);
5246
5247 tmp = RREG32(RLC_AUTO_PG_CTRL);
5248 tmp |= AUTO_PG_EN;
5249 WREG32(RLC_AUTO_PG_CTRL, tmp);
5250 } else {
5251 tmp = RREG32(RLC_AUTO_PG_CTRL);
5252 tmp &= ~AUTO_PG_EN;
5253 WREG32(RLC_AUTO_PG_CTRL, tmp);
5254
5255 tmp = RREG32(DB_RENDER_CONTROL);
5256 }
5257}
5258
5259static void si_init_gfx_cgpg(struct radeon_device *rdev)
5260{
5261 u32 tmp;
5262
5263 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5264
5265 tmp = RREG32(RLC_PG_CNTL);
5266 tmp |= GFX_PG_SRC;
5267 WREG32(RLC_PG_CNTL, tmp);
5268
5269 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5270
5271 tmp = RREG32(RLC_AUTO_PG_CTRL);
5272
5273 tmp &= ~GRBM_REG_SGIT_MASK;
5274 tmp |= GRBM_REG_SGIT(0x700);
5275 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5276 WREG32(RLC_AUTO_PG_CTRL, tmp);
5277}
5278
ba19031a 5279static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
f8f84ac5
AD		 5280{
5281 u32 mask = 0, tmp, tmp1;
5282 int i;
5283
5284 si_select_se_sh(rdev, se, sh);
5285 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5286 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5287 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5288
5289 tmp &= 0xffff0000;
5290
5291 tmp |= tmp1;
5292 tmp >>= 16;
5293
5294 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5295 mask <<= 1;
5296 mask |= 1;
5297 }
5298
5299 return (~tmp) & mask;
5300}
5301
5302static void si_init_ao_cu_mask(struct radeon_device *rdev)
5303{
5304 u32 i, j, k, active_cu_number = 0;
5305 u32 mask, counter, cu_bitmap;
5306 u32 tmp = 0;
5307
5308 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5309 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5310 mask = 1;
5311 cu_bitmap = 0;
5312 counter = 0;
5313 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
ba19031a 5314 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
f8f84ac5
AD		 5315 if (counter < 2)
5316 cu_bitmap |= mask;
5317 counter++;
5318 }
5319 mask <<= 1;
5320 }
5321
5322 active_cu_number += counter;
5323 tmp |= (cu_bitmap << (i * 16 + j * 8));
5324 }
5325 }
5326
5327 WREG32(RLC_PG_AO_CU_MASK, tmp);
5328
5329 tmp = RREG32(RLC_MAX_PG_CU);
5330 tmp &= ~MAX_PU_CU_MASK;
5331 tmp |= MAX_PU_CU(active_cu_number);
5332 WREG32(RLC_MAX_PG_CU, tmp);
5333}
5334
5335static void si_enable_cgcg(struct radeon_device *rdev,
5336 bool enable)
5337{
5338 u32 data, orig, tmp;
5339
5340 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5341
e16866ec 5342 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5594a558 5343 si_enable_gui_idle_interrupt(rdev, true);
f8f84ac5 5344
f8f84ac5
AD		 5345 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5346
5347 tmp = si_halt_rlc(rdev);
5348
5349 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5350 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5351 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5352
5353 si_wait_for_rlc_serdes(rdev);
5354
5355 si_update_rlc(rdev, tmp);
5356
5357 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5358
5359 data |= CGCG_EN | CGLS_EN;
5360 } else {
5594a558
AD		 5361 si_enable_gui_idle_interrupt(rdev, false);
5362
f8f84ac5
AD		 5363 RREG32(CB_CGTT_SCLK_CTRL);
5364 RREG32(CB_CGTT_SCLK_CTRL);
5365 RREG32(CB_CGTT_SCLK_CTRL);
5366 RREG32(CB_CGTT_SCLK_CTRL);
5367
5368 data &= ~(CGCG_EN | CGLS_EN);
5369 }
5370
5371 if (orig != data)
5372 WREG32(RLC_CGCG_CGLS_CTRL, data);
5373}
5374
5375static void si_enable_mgcg(struct radeon_device *rdev,
5376 bool enable)
5377{
5378 u32 data, orig, tmp = 0;
5379
e16866ec 5380 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
f8f84ac5
AD		 5381 orig = data = RREG32(CGTS_SM_CTRL_REG);
5382 data = 0x96940200;
5383 if (orig != data)
5384 WREG32(CGTS_SM_CTRL_REG, data);
5385
e16866ec
AD		 5386 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5387 orig = data = RREG32(CP_MEM_SLP_CNTL);
5388 data |= CP_MEM_LS_EN;
5389 if (orig != data)
5390 WREG32(CP_MEM_SLP_CNTL, data);
5391 }
f8f84ac5
AD		 5392
5393 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5394 data &= 0xffffffc0;
5395 if (orig != data)
5396 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5397
5398 tmp = si_halt_rlc(rdev);
5399
5400 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5401 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5402 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5403
5404 si_update_rlc(rdev, tmp);
5405 } else {
5406 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5407 data |= 0x00000003;
5408 if (orig != data)
5409 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5410
5411 data = RREG32(CP_MEM_SLP_CNTL);
5412 if (data & CP_MEM_LS_EN) {
5413 data &= ~CP_MEM_LS_EN;
5414 WREG32(CP_MEM_SLP_CNTL, data);
5415 }
5416 orig = data = RREG32(CGTS_SM_CTRL_REG);
5417 data |= LS_OVERRIDE | OVERRIDE;
5418 if (orig != data)
5419 WREG32(CGTS_SM_CTRL_REG, data);
5420
5421 tmp = si_halt_rlc(rdev);
5422
5423 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5424 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5425 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5426
5427 si_update_rlc(rdev, tmp);
5428 }
5429}
5430
5431static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5432 bool enable)
5433{
5434 u32 orig, data, tmp;
5435
e16866ec 5436 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
f8f84ac5
AD		 5437 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5438 tmp |= 0x3fff;
5439 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5440
5441 orig = data = RREG32(UVD_CGC_CTRL);
5442 data |= DCM;
5443 if (orig != data)
5444 WREG32(UVD_CGC_CTRL, data);
5445
5446 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5447 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5448 } else {
5449 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5450 tmp &= ~0x3fff;
5451 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5452
5453 orig = data = RREG32(UVD_CGC_CTRL);
5454 data &= ~DCM;
5455 if (orig != data)
5456 WREG32(UVD_CGC_CTRL, data);
5457
5458 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5459 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5460 }
5461}
5462
5463static const u32 mc_cg_registers[] =
5464{
5465 MC_HUB_MISC_HUB_CG,
5466 MC_HUB_MISC_SIP_CG,
5467 MC_HUB_MISC_VM_CG,
5468 MC_XPB_CLK_GAT,
5469 ATC_MISC_CG,
5470 MC_CITF_MISC_WR_CG,
5471 MC_CITF_MISC_RD_CG,
5472 MC_CITF_MISC_VM_CG,
5473 VM_L2_CG,
5474};
5475
5476static void si_enable_mc_ls(struct radeon_device *rdev,
5477 bool enable)
5478{
5479 int i;
5480 u32 orig, data;
5481
5482 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5483 orig = data = RREG32(mc_cg_registers[i]);
e16866ec 5484 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
f8f84ac5
AD		 5485 data |= MC_LS_ENABLE;
5486 else
5487 data &= ~MC_LS_ENABLE;
5488 if (data != orig)
5489 WREG32(mc_cg_registers[i], data);
5490 }
5491}
5492
e16866ec
AD		 5493static void si_enable_mc_mgcg(struct radeon_device *rdev,
5494 bool enable)
f8f84ac5 5495{
e16866ec
AD		 5496 int i;
5497 u32 orig, data;
5498
5499 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5500 orig = data = RREG32(mc_cg_registers[i]);
5501 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5502 data |= MC_CG_ENABLE;
5503 else
5504 data &= ~MC_CG_ENABLE;
5505 if (data != orig)
5506 WREG32(mc_cg_registers[i], data);
f8f84ac5
AD		 5507 }
5508}
5509
e16866ec
AD		 5510static void si_enable_dma_mgcg(struct radeon_device *rdev,
5511 bool enable)
f8f84ac5 5512{
e16866ec
AD		 5513 u32 orig, data, offset;
5514 int i;
f8f84ac5 5515
e16866ec
AD		 5516 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5517 for (i = 0; i < 2; i++) {
5518 if (i == 0)
5519 offset = DMA0_REGISTER_OFFSET;
5520 else
5521 offset = DMA1_REGISTER_OFFSET;
5522 orig = data = RREG32(DMA_POWER_CNTL + offset);
5523 data &= ~MEM_POWER_OVERRIDE;
5524 if (data != orig)
5525 WREG32(DMA_POWER_CNTL + offset, data);
5526 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5527 }
5528 } else {
5529 for (i = 0; i < 2; i++) {
5530 if (i == 0)
5531 offset = DMA0_REGISTER_OFFSET;
5532 else
5533 offset = DMA1_REGISTER_OFFSET;
5534 orig = data = RREG32(DMA_POWER_CNTL + offset);
5535 data |= MEM_POWER_OVERRIDE;
5536 if (data != orig)
5537 WREG32(DMA_POWER_CNTL + offset, data);
5538
5539 orig = data = RREG32(DMA_CLK_CTRL + offset);
5540 data = 0xff000000;
5541 if (data != orig)
5542 WREG32(DMA_CLK_CTRL + offset, data);
5543 }
5544 }
f8f84ac5
AD		 5545}
5546
e16866ec
AD		 5547static void si_enable_bif_mgls(struct radeon_device *rdev,
5548 bool enable)
f8f84ac5 5549{
e16866ec 5550 u32 orig, data;
f8f84ac5 5551
e16866ec 5552 orig = data = RREG32_PCIE(PCIE_CNTL2);
f8f84ac5 5553
e16866ec
AD		 5554 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5555 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5556 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5557 else
5558 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5559 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5560
5561 if (orig != data)
5562 WREG32_PCIE(PCIE_CNTL2, data);
f8f84ac5
AD		 5563}
5564
e16866ec
AD		 5565static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5566 bool enable)
f8f84ac5 5567{
e16866ec 5568 u32 orig, data;
f8f84ac5 5569
e16866ec 5570 orig = data = RREG32(HDP_HOST_PATH_CNTL);
f8f84ac5 5571
e16866ec
AD		 5572 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5573 data &= ~CLOCK_GATING_DIS;
5574 else
5575 data |= CLOCK_GATING_DIS;
5576
5577 if (orig != data)
5578 WREG32(HDP_HOST_PATH_CNTL, data);
f8f84ac5
AD		 5579}
5580
e16866ec
AD		 5581static void si_enable_hdp_ls(struct radeon_device *rdev,
5582 bool enable)
347e7592 5583{
e16866ec 5584 u32 orig, data;
347e7592 5585
e16866ec 5586 orig = data = RREG32(HDP_MEM_POWER_LS);
347e7592 5587
e16866ec
AD		 5588 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5589 data |= HDP_LS_ENABLE;
5590 else
5591 data &= ~HDP_LS_ENABLE;
347e7592 5592
e16866ec
AD		 5593 if (orig != data)
5594 WREG32(HDP_MEM_POWER_LS, data);
347e7592
AD		 5595}
5596
68e3a092
AD		 5597static void si_update_cg(struct radeon_device *rdev,
5598 u32 block, bool enable)
347e7592 5599{
e16866ec 5600 if (block & RADEON_CG_BLOCK_GFX) {
811e4d58 5601 si_enable_gui_idle_interrupt(rdev, false);
e16866ec
AD		 5602 /* order matters! */
5603 if (enable) {
5604 si_enable_mgcg(rdev, true);
5605 si_enable_cgcg(rdev, true);
5606 } else {
5607 si_enable_cgcg(rdev, false);
5608 si_enable_mgcg(rdev, false);
347e7592 5609 }
811e4d58 5610 si_enable_gui_idle_interrupt(rdev, true);
347e7592
AD		 5611 }
5612
e16866ec
AD		 5613 if (block & RADEON_CG_BLOCK_MC) {
5614 si_enable_mc_mgcg(rdev, enable);
5615 si_enable_mc_ls(rdev, enable);
347e7592 5616 }
e16866ec
AD		 5617
5618 if (block & RADEON_CG_BLOCK_SDMA) {
5619 si_enable_dma_mgcg(rdev, enable);
347e7592
AD		 5620 }
5621
e16866ec
AD		 5622 if (block & RADEON_CG_BLOCK_BIF) {
5623 si_enable_bif_mgls(rdev, enable);
6d8cf000 5624 }
6d8cf000 5625
e16866ec
AD		 5626 if (block & RADEON_CG_BLOCK_UVD) {
5627 if (rdev->has_uvd) {
5628 si_enable_uvd_mgcg(rdev, enable);
bd8cd539
AD		 5629 }
5630 }
bd8cd539 5631
e16866ec
AD		 5632 if (block & RADEON_CG_BLOCK_HDP) {
5633 si_enable_hdp_mgcg(rdev, enable);
5634 si_enable_hdp_ls(rdev, enable);
347e7592 5635 }
e16866ec 5636}
f8f84ac5
AD		 5637
5638static void si_init_cg(struct radeon_device *rdev)
5639{
e16866ec
AD		 5640 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5641 RADEON_CG_BLOCK_MC |
5642 RADEON_CG_BLOCK_SDMA |
5643 RADEON_CG_BLOCK_BIF |
5644 RADEON_CG_BLOCK_HDP), true);
b2d70917 5645 if (rdev->has_uvd) {
e16866ec 5646 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
f8f84ac5 5647 si_init_uvd_internal_cg(rdev);
347e7592 5648 }
f8f84ac5 5649}
bd8cd539 5650
f8f84ac5
AD		 5651static void si_fini_cg(struct radeon_device *rdev)
5652{
0116e1ef 5653 if (rdev->has_uvd) {
e16866ec 5654 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
347e7592 5655 }
e16866ec
AD		 5656 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5657 RADEON_CG_BLOCK_MC |
5658 RADEON_CG_BLOCK_SDMA |
5659 RADEON_CG_BLOCK_BIF |
5660 RADEON_CG_BLOCK_HDP), false);
5661}
5662
59a82d0e
AD		 5663u32 si_get_csb_size(struct radeon_device *rdev)
5664{
5665 u32 count = 0;
5666 const struct cs_section_def *sect = NULL;
5667 const struct cs_extent_def *ext = NULL;
5668
5669 if (rdev->rlc.cs_data == NULL)
5670 return 0;
5671
5672 /* begin clear state */
5673 count += 2;
5674 /* context control state */
5675 count += 3;
5676
5677 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5678 for (ext = sect->section; ext->extent != NULL; ++ext) {
5679 if (sect->id == SECT_CONTEXT)
5680 count += 2 + ext->reg_count;
5681 else
5682 return 0;
5683 }
bd8cd539 5684 }
59a82d0e
AD		 5685 /* pa_sc_raster_config */
5686 count += 3;
5687 /* end clear state */
5688 count += 2;
5689 /* clear state */
5690 count += 2;
5691
5692 return count;
5693}
5694
5695void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5696{
5697 u32 count = 0, i;
5698 const struct cs_section_def *sect = NULL;
5699 const struct cs_extent_def *ext = NULL;
5700
5701 if (rdev->rlc.cs_data == NULL)
5702 return;
5703 if (buffer == NULL)
5704 return;
5705
6ba81e53
AD		 5706 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5707 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
59a82d0e 5708
6ba81e53
AD		 5709 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5710 buffer[count++] = cpu_to_le32(0x80000000);
5711 buffer[count++] = cpu_to_le32(0x80000000);
59a82d0e
AD		 5712
5713 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5714 for (ext = sect->section; ext->extent != NULL; ++ext) {
5715 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 5716 buffer[count++] =
5717 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5718 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
59a82d0e 5719 for (i = 0; i < ext->reg_count; i++)
6ba81e53 5720 buffer[count++] = cpu_to_le32(ext->extent[i]);
59a82d0e
AD		 5721 } else {
5722 return;
bd8cd539 5723 }
bd8cd539
AD		 5724 }
5725 }
bd8cd539 5726
6ba81e53
AD		 5727 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5728 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
59a82d0e
AD		 5729 switch (rdev->family) {
5730 case CHIP_TAHITI:
5731 case CHIP_PITCAIRN:
6ba81e53 5732 buffer[count++] = cpu_to_le32(0x2a00126a);
59a82d0e
AD		 5733 break;
5734 case CHIP_VERDE:
6ba81e53 5735 buffer[count++] = cpu_to_le32(0x0000124a);
59a82d0e
AD		 5736 break;
5737 case CHIP_OLAND:
6ba81e53 5738 buffer[count++] = cpu_to_le32(0x00000082);
59a82d0e
AD		 5739 break;
5740 case CHIP_HAINAN:
6ba81e53 5741 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5742 break;
5743 default:
6ba81e53 5744 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5745 break;
5746 }
5747
6ba81e53
AD		 5748 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5749 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
347e7592 5750
6ba81e53
AD		 5751 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5752 buffer[count++] = cpu_to_le32(0);
59a82d0e
AD		 5753}
5754
f8f84ac5
AD		 5755static void si_init_pg(struct radeon_device *rdev)
5756{
0116e1ef
AD		 5757 if (rdev->pg_flags) {
5758 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5759 si_init_dma_pg(rdev);
0116e1ef 5760 }
f8f84ac5 5761 si_init_ao_cu_mask(rdev);
2b19d17f 5762 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
0116e1ef 5763 si_init_gfx_cgpg(rdev);
aa34dba8
AD		 5764 } else {
5765 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5766 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
0116e1ef 5767 }
ca6ebb39
AD		 5768 si_enable_dma_pg(rdev, true);
5769 si_enable_gfx_cgpg(rdev, true);
f8f84ac5
AD		 5770 } else {
5771 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5772 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5773 }
5774}
5775
5776static void si_fini_pg(struct radeon_device *rdev)
5777{
0116e1ef 5778 if (rdev->pg_flags) {
ca6ebb39
AD		 5779 si_enable_dma_pg(rdev, false);
5780 si_enable_gfx_cgpg(rdev, false);
f8f84ac5 5781 }
347e7592
AD		 5782}
5783
347e7592
AD		 5784/*
5785 * RLC
5786 */
866d83de 5787void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5788{
f8f84ac5 5789 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5790
f8f84ac5
AD		 5791 tmp |= SOFT_RESET_RLC;
5792 WREG32(GRBM_SOFT_RESET, tmp);
5793 udelay(50);
5794 tmp &= ~SOFT_RESET_RLC;
5795 WREG32(GRBM_SOFT_RESET, tmp);
5796 udelay(50);
d719cef3
AD		 5797}
5798
347e7592
AD		 5799static void si_rlc_stop(struct radeon_device *rdev)
5800{
5801 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5802
5803 si_enable_gui_idle_interrupt(rdev, false);
5804
5805 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5806}
5807
5808static void si_rlc_start(struct radeon_device *rdev)
5809{
5810 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5811
5812 si_enable_gui_idle_interrupt(rdev, true);
5813
5814 udelay(50);
5815}
5816
5817static bool si_lbpw_supported(struct radeon_device *rdev)
5818{
5819 u32 tmp;
5820
5821 /* Enable LBPW only for DDR3 */
5822 tmp = RREG32(MC_SEQ_MISC0);
5823 if ((tmp & 0xF0000000) == 0xB0000000)
5824 return true;
5825 return false;
5826}
5827
5828static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5829{
5830 u32 tmp;
5831
5832 tmp = RREG32(RLC_LB_CNTL);
5833 if (enable)
5834 tmp |= LOAD_BALANCE_ENABLE;
5835 else
5836 tmp &= ~LOAD_BALANCE_ENABLE;
5837 WREG32(RLC_LB_CNTL, tmp);
5838
5839 if (!enable) {
5840 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5841 WREG32(SPI_LB_CU_MASK, 0x00ff);
5842 }
347e7592
AD		 5843}
5844
5845static int si_rlc_resume(struct radeon_device *rdev)
5846{
5847 u32 i;
347e7592
AD		 5848
5849 if (!rdev->rlc_fw)
5850 return -EINVAL;
5851
5852 si_rlc_stop(rdev);
5853
f8f84ac5
AD		 5854 si_rlc_reset(rdev);
5855
5856 si_init_pg(rdev);
5857
5858 si_init_cg(rdev);
5859
347e7592
AD		 5860 WREG32(RLC_RL_BASE, 0);
5861 WREG32(RLC_RL_SIZE, 0);
5862 WREG32(RLC_LB_CNTL, 0);
5863 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5864 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5865 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5866
347e7592
AD		 5867 WREG32(RLC_MC_CNTL, 0);
5868 WREG32(RLC_UCODE_CNTL, 0);
5869
629bd33c
AD		 5870 if (rdev->new_fw) {
5871 const struct rlc_firmware_header_v1_0 *hdr =
5872 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5873 u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5874 const __le32 *fw_data = (const __le32 *)
5875 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5876
5877 radeon_ucode_print_rlc_hdr(&hdr->header);
5878
5879 for (i = 0; i < fw_size; i++) {
5880 WREG32(RLC_UCODE_ADDR, i);
5881 WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5882 }
5883 } else {
5884 const __be32 *fw_data =
5885 (const __be32 *)rdev->rlc_fw->data;
5886 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5887 WREG32(RLC_UCODE_ADDR, i);
5888 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5889 }
347e7592
AD		 5890 }
5891 WREG32(RLC_UCODE_ADDR, 0);
5892
d719cef3
AD		 5893 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5894
347e7592
AD		 5895 si_rlc_start(rdev);
5896
5897 return 0;
5898}
5899
25a857fb
AD		 5900static void si_enable_interrupts(struct radeon_device *rdev)
5901{
5902 u32 ih_cntl = RREG32(IH_CNTL);
5903 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5904
5905 ih_cntl |= ENABLE_INTR;
5906 ih_rb_cntl |= IH_RB_ENABLE;
5907 WREG32(IH_CNTL, ih_cntl);
5908 WREG32(IH_RB_CNTL, ih_rb_cntl);
5909 rdev->ih.enabled = true;
5910}
5911
5912static void si_disable_interrupts(struct radeon_device *rdev)
5913{
5914 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5915 u32 ih_cntl = RREG32(IH_CNTL);
5916
5917 ih_rb_cntl &= ~IH_RB_ENABLE;
5918 ih_cntl &= ~ENABLE_INTR;
5919 WREG32(IH_RB_CNTL, ih_rb_cntl);
5920 WREG32(IH_CNTL, ih_cntl);
5921 /* set rptr, wptr to 0 */
5922 WREG32(IH_RB_RPTR, 0);
5923 WREG32(IH_RB_WPTR, 0);
5924 rdev->ih.enabled = false;
25a857fb
AD		 5925 rdev->ih.rptr = 0;
5926}
5927
5928static void si_disable_interrupt_state(struct radeon_device *rdev)
5929{
4cd096dd 5930 int i;
25a857fb
AD		 5931 u32 tmp;
5932
811e4d58
AD		 5933 tmp = RREG32(CP_INT_CNTL_RING0) &
5934 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5935 WREG32(CP_INT_CNTL_RING0, tmp);
25a857fb
AD		 5936 WREG32(CP_INT_CNTL_RING1, 0);
5937 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5938 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5939 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5940 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5941 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5942 WREG32(GRBM_INT_CNTL, 0);
18ad01ef 5943 WREG32(SRBM_INT_CNTL, 0);
4cd096dd
L		 5944 for (i = 0; i < rdev->num_crtc; i++)
5945 WREG32(INT_MASK + crtc_offsets[i], 0);
98990fae
L		 5946 for (i = 0; i < rdev->num_crtc; i++)
5947 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
25a857fb 5948
5153550a 5949 if (!ASIC_IS_NODCE(rdev)) {
e9a321c6 5950 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5153550a 5951
4cd096dd
L		 5952 for (i = 0; i < 6; i++)
5953 WREG32_AND(DC_HPDx_INT_CONTROL(i),
5954 DC_HPDx_INT_POLARITY);
5153550a 5955 }
25a857fb
AD		 5956}
5957
5958static int si_irq_init(struct radeon_device *rdev)
5959{
5960 int ret = 0;
5961 int rb_bufsz;
5962 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5963
5964 /* allocate ring */
5965 ret = r600_ih_ring_alloc(rdev);
5966 if (ret)
5967 return ret;
5968
5969 /* disable irqs */
5970 si_disable_interrupts(rdev);
5971
5972 /* init rlc */
5973 ret = si_rlc_resume(rdev);
5974 if (ret) {
5975 r600_ih_ring_fini(rdev);
5976 return ret;
5977 }
5978
5979 /* setup interrupt control */
62d91dd2
SB		 5980 /* set dummy read address to dummy page address */
5981 WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
25a857fb
AD		 5982 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5983 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5984 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5985 */
5986 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5987 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5988 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5989 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5990
5991 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 5992 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
25a857fb
AD		 5993
5994 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5995 IH_WPTR_OVERFLOW_CLEAR |
5996 (rb_bufsz << 1));
5997
5998 if (rdev->wb.enabled)
5999 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6000
6001 /* set the writeback address whether it's enabled or not */
6002 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6003 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6004
6005 WREG32(IH_RB_CNTL, ih_rb_cntl);
6006
6007 /* set rptr, wptr to 0 */
6008 WREG32(IH_RB_RPTR, 0);
6009 WREG32(IH_RB_WPTR, 0);
6010
6011 /* Default settings for IH_CNTL (disabled at first) */
6012 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6013 /* RPTR_REARM only works if msi's are enabled */
6014 if (rdev->msi_enabled)
6015 ih_cntl |= RPTR_REARM;
6016 WREG32(IH_CNTL, ih_cntl);
6017
6018 /* force the active interrupt state to all disabled */
6019 si_disable_interrupt_state(rdev);
6020
2099810f
DA		 6021 pci_set_master(rdev->pdev);
6022
25a857fb
AD		 6023 /* enable irqs */
6024 si_enable_interrupts(rdev);
6025
6026 return ret;
6027}
6028
4cd096dd 6029/* The order we write back each register here is important */
25a857fb
AD		 6030int si_irq_set(struct radeon_device *rdev)
6031{
4cd096dd 6032 int i;
811e4d58 6033 u32 cp_int_cntl;
25a857fb 6034 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
25a857fb 6035 u32 grbm_int_cntl = 0;
8c5fd7ef 6036 u32 dma_cntl, dma_cntl1;
a9e61410 6037 u32 thermal_int = 0;
25a857fb
AD		 6038
6039 if (!rdev->irq.installed) {
6040 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6041 return -EINVAL;
6042 }
6043 /* don't enable anything if the ih is disabled */
6044 if (!rdev->ih.enabled) {
6045 si_disable_interrupts(rdev);
6046 /* force the active interrupt state to all disabled */
6047 si_disable_interrupt_state(rdev);
6048 return 0;
6049 }
6050
811e4d58
AD		 6051 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6052 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6053
8c5fd7ef
AD		 6054 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6055 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6056
a9e61410
AD		 6057 thermal_int = RREG32(CG_THERMAL_INT) &
6058 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6059
25a857fb 6060 /* enable CP interrupts on all rings */
736fc37f 6061 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 6062 DRM_DEBUG("si_irq_set: sw int gfx\n");
6063 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6064 }
736fc37f 6065 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 6066 DRM_DEBUG("si_irq_set: sw int cp1\n");
6067 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6068 }
736fc37f 6069 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 6070 DRM_DEBUG("si_irq_set: sw int cp2\n");
6071 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6072 }
8c5fd7ef
AD		 6073 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6074 DRM_DEBUG("si_irq_set: sw int dma\n");
6075 dma_cntl |= TRAP_ENABLE;
6076 }
6077
6078 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6079 DRM_DEBUG("si_irq_set: sw int dma1\n");
6080 dma_cntl1 |= TRAP_ENABLE;
6081 }
25a857fb
AD		 6082
6083 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6084 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6085 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6086
8c5fd7ef
AD		 6087 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6088 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6089
25a857fb
AD		 6090 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6091
a9e61410
AD		 6092 if (rdev->irq.dpm_thermal) {
6093 DRM_DEBUG("dpm thermal\n");
6094 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6095 }
6096
4cd096dd
L		 6097 for (i = 0; i < rdev->num_crtc; i++) {
6098 radeon_irq_kms_set_irq_n_enabled(
6099 rdev, INT_MASK + crtc_offsets[i], VBLANK_INT_MASK,
6100 rdev->irq.crtc_vblank_int[i] ||
6101 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
25a857fb
AD		 6102 }
6103
98990fae
L		 6104 for (i = 0; i < rdev->num_crtc; i++)
6105 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
25a857fb 6106
5153550a 6107 if (!ASIC_IS_NODCE(rdev)) {
4cd096dd
L		 6108 for (i = 0; i < 6; i++) {
6109 radeon_irq_kms_set_irq_n_enabled(
6110 rdev, DC_HPDx_INT_CONTROL(i),
6111 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
6112 rdev->irq.hpd[i], "HPD", i);
6113 }
5153550a 6114 }
25a857fb 6115
a9e61410
AD		 6116 WREG32(CG_THERMAL_INT, thermal_int);
6117
0586915e
AD		 6118 /* posting read */
6119 RREG32(SRBM_STATUS);
6120
25a857fb
AD		 6121 return 0;
6122}
6123
4cd096dd 6124/* The order we write back each register here is important */
25a857fb
AD		 6125static inline void si_irq_ack(struct radeon_device *rdev)
6126{
98990fae 6127 int i, j;
4cd096dd 6128 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
98990fae 6129 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
25a857fb 6130
5153550a
AD		 6131 if (ASIC_IS_NODCE(rdev))
6132 return;
6133
98990fae 6134 for (i = 0; i < 6; i++) {
4cd096dd 6135 disp_int[i] = RREG32(si_disp_int_status[i]);
98990fae
L		 6136 if (i < rdev->num_crtc)
6137 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
25a857fb
AD		 6138 }
6139
98990fae
L		 6140 /* We write back each interrupt register in pairs of two */
6141 for (i = 0; i < rdev->num_crtc; i += 2) {
6142 for (j = i; j < (i + 2); j++) {
6143 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
6144 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
6145 GRPH_PFLIP_INT_CLEAR);
6146 }
25a857fb 6147
98990fae
L		 6148 for (j = i; j < (i + 2); j++) {
6149 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
6150 WREG32(VBLANK_STATUS + crtc_offsets[j],
6151 VBLANK_ACK);
6152 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
6153 WREG32(VLINE_STATUS + crtc_offsets[j],
6154 VLINE_ACK);
6155 }
4cd096dd
L		 6156 }
6157
6158 for (i = 0; i < 6; i++) {
6159 if (disp_int[i] & DC_HPD1_INTERRUPT)
6160 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
6161 }
6162
6163 for (i = 0; i < 6; i++) {
6164 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
6165 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
47f2467f 6166 }
25a857fb
AD		 6167}
6168
6169static void si_irq_disable(struct radeon_device *rdev)
6170{
6171 si_disable_interrupts(rdev);
6172 /* Wait and acknowledge irq */
6173 mdelay(1);
6174 si_irq_ack(rdev);
6175 si_disable_interrupt_state(rdev);
6176}
6177
6178static void si_irq_suspend(struct radeon_device *rdev)
6179{
6180 si_irq_disable(rdev);
6181 si_rlc_stop(rdev);
6182}
6183
9b136d51
AD		 6184static void si_irq_fini(struct radeon_device *rdev)
6185{
6186 si_irq_suspend(rdev);
6187 r600_ih_ring_fini(rdev);
6188}
6189
25a857fb
AD		 6190static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6191{
6192 u32 wptr, tmp;
6193
6194 if (rdev->wb.enabled)
6195 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6196 else
6197 wptr = RREG32(IH_RB_WPTR);
6198
6199 if (wptr & RB_OVERFLOW) {
11bab0ae 6200 wptr &= ~RB_OVERFLOW;
25a857fb
AD		 6201 /* When a ring buffer overflow happen start parsing interrupt
6202 * from the last not overwritten vector (wptr + 16). Hopefully
6203 * this should allow us to catchup.
6204 */
6cc2fda2
MD		 6205 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6206 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
25a857fb
AD		 6207 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6208 tmp = RREG32(IH_RB_CNTL);
6209 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6210 WREG32(IH_RB_CNTL, tmp);
6211 }
6212 return (wptr & rdev->ih.ptr_mask);
6213}
6214
6215/* SI IV Ring
6216 * Each IV ring entry is 128 bits:
6217 * [7:0] - interrupt source id
6218 * [31:8] - reserved
6219 * [59:32] - interrupt source data
6220 * [63:60] - reserved
6221 * [71:64] - RINGID
6222 * [79:72] - VMID
6223 * [127:80] - reserved
6224 */
6225int si_irq_process(struct radeon_device *rdev)
6226{
4cd096dd
L		 6227 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6228 u32 crtc_idx, hpd_idx;
6229 u32 mask;
25a857fb
AD		 6230 u32 wptr;
6231 u32 rptr;
6232 u32 src_id, src_data, ring_id;
6233 u32 ring_index;
25a857fb 6234 bool queue_hotplug = false;
47f2467f 6235 bool queue_dp = false;
a9e61410 6236 bool queue_thermal = false;
fbf6dc7a 6237 u32 status, addr;
4cd096dd 6238 const char *event_name;
25a857fb
AD		 6239
6240 if (!rdev->ih.enabled || rdev->shutdown)
6241 return IRQ_NONE;
6242
6243 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 6244
6245restart_ih:
6246 /* is somebody else already processing irqs? */
6247 if (atomic_xchg(&rdev->ih.lock, 1))
6248 return IRQ_NONE;
6249
25a857fb
AD		 6250 rptr = rdev->ih.rptr;
6251 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6252
25a857fb
AD		 6253 /* Order reading of wptr vs. reading of IH ring data */
6254 rmb();
6255
6256 /* display interrupts */
6257 si_irq_ack(rdev);
6258
25a857fb
AD		 6259 while (rptr != wptr) {
6260 /* wptr/rptr are in bytes! */
6261 ring_index = rptr / 4;
6262 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6263 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6264 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6265
6266 switch (src_id) {
6267 case 1: /* D1 vblank/vline */
25a857fb 6268 case 2: /* D2 vblank/vline */
25a857fb 6269 case 3: /* D3 vblank/vline */
25a857fb 6270 case 4: /* D4 vblank/vline */
25a857fb 6271 case 5: /* D5 vblank/vline */
4cd096dd
L		 6272 case 6: /* D6 vblank/vline */
6273 crtc_idx = src_id - 1;
07f18f0b 6274
4cd096dd
L		 6275 if (src_data == 0) { /* vblank */
6276 mask = LB_D1_VBLANK_INTERRUPT;
6277 event_name = "vblank";
6278
6279 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
6280 drm_handle_vblank(rdev->ddev, crtc_idx);
07f18f0b
MK		 6281 rdev->pm.vblank_sync = true;
6282 wake_up(&rdev->irq.vblank_queue);
25a857fb 6283 }
4cd096dd
L		 6284 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
6285 radeon_crtc_handle_vblank(rdev,
6286 crtc_idx);
6287 }
07f18f0b 6288
4cd096dd
L		 6289 } else if (src_data == 1) { /* vline */
6290 mask = LB_D1_VLINE_INTERRUPT;
6291 event_name = "vline";
6292 } else {
6293 DRM_DEBUG("Unhandled interrupt: %d %d\n",
6294 src_id, src_data);
25a857fb
AD		 6295 break;
6296 }
07f18f0b 6297
4cd096dd
L		 6298 if (!(disp_int[crtc_idx] & mask)) {
6299 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
6300 crtc_idx + 1, event_name);
6301 }
07f18f0b 6302
4cd096dd
L		 6303 disp_int[crtc_idx] &= ~mask;
6304 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
07f18f0b 6305
25a857fb 6306 break;
f5d636d2
CK		 6307 case 8: /* D1 page flip */
6308 case 10: /* D2 page flip */
6309 case 12: /* D3 page flip */
6310 case 14: /* D4 page flip */
6311 case 16: /* D5 page flip */
6312 case 18: /* D6 page flip */
6313 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 6314 if (radeon_use_pflipirq > 0)
6315 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 6316 break;
25a857fb 6317 case 42: /* HPD hotplug */
4cd096dd
L		 6318 if (src_data <= 5) {
6319 hpd_idx = src_data;
6320 mask = DC_HPD1_INTERRUPT;
07f18f0b 6321 queue_hotplug = true;
4cd096dd 6322 event_name = "HPD";
07f18f0b 6323
4cd096dd
L		 6324 } else if (src_data <= 11) {
6325 hpd_idx = src_data - 6;
6326 mask = DC_HPD1_RX_INTERRUPT;
07f18f0b 6327 queue_dp = true;
4cd096dd 6328 event_name = "HPD_RX";
07f18f0b 6329
4cd096dd
L		 6330 } else {
6331 DRM_DEBUG("Unhandled interrupt: %d %d\n",
6332 src_id, src_data);
47f2467f 6333 break;
4cd096dd 6334 }
07f18f0b 6335
4cd096dd
L		 6336 if (!(disp_int[hpd_idx] & mask))
6337 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
07f18f0b 6338
4cd096dd
L		 6339 disp_int[hpd_idx] &= ~mask;
6340 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
25a857fb 6341 break;
18ad01ef
CK		 6342 case 96:
6343 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6344 WREG32(SRBM_INT_ACK, 0x1);
6345 break;
b927e1c2
CK		 6346 case 124: /* UVD */
6347 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6348 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6349 break;
ae133a11
CK		 6350 case 146:
6351 case 147:
fbf6dc7a
AD		 6352 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6353 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
9b7d786b
CK		 6354 /* reset addr and status */
6355 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6356 if (addr == 0x0 && status == 0x0)
6357 break;
ae133a11
CK		 6358 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6359 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
fbf6dc7a 6360 addr);
ae133a11 6361 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
fbf6dc7a
AD		 6362 status);
6363 si_vm_decode_fault(rdev, status, addr);
ae133a11 6364 break;
25a857fb
AD		 6365 case 176: /* RINGID0 CP_INT */
6366 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6367 break;
6368 case 177: /* RINGID1 CP_INT */
6369 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6370 break;
6371 case 178: /* RINGID2 CP_INT */
6372 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6373 break;
6374 case 181: /* CP EOP event */
6375 DRM_DEBUG("IH: CP EOP\n");
6376 switch (ring_id) {
6377 case 0:
6378 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6379 break;
6380 case 1:
6381 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6382 break;
6383 case 2:
6384 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6385 break;
6386 }
6387 break;
8c5fd7ef
AD		 6388 case 224: /* DMA trap event */
6389 DRM_DEBUG("IH: DMA trap\n");
6390 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6391 break;
a9e61410
AD		 6392 case 230: /* thermal low to high */
6393 DRM_DEBUG("IH: thermal low to high\n");
6394 rdev->pm.dpm.thermal.high_to_low = false;
6395 queue_thermal = true;
6396 break;
6397 case 231: /* thermal high to low */
6398 DRM_DEBUG("IH: thermal high to low\n");
6399 rdev->pm.dpm.thermal.high_to_low = true;
6400 queue_thermal = true;
6401 break;
25a857fb
AD		 6402 case 233: /* GUI IDLE */
6403 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6404 break;
8c5fd7ef
AD		 6405 case 244: /* DMA trap event */
6406 DRM_DEBUG("IH: DMA1 trap\n");
6407 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6408 break;
25a857fb
AD		 6409 default:
6410 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6411 break;
6412 }
6413
6414 /* wptr/rptr are in bytes! */
6415 rptr += 16;
6416 rptr &= rdev->ih.ptr_mask;
f55e03b9 6417 WREG32(IH_RB_RPTR, rptr);
25a857fb 6418 }
47f2467f
DA		 6419 if (queue_dp)
6420 schedule_work(&rdev->dp_work);
25a857fb 6421 if (queue_hotplug)
cb5d4166 6422 schedule_delayed_work(&rdev->hotplug_work, 0);
a9e61410
AD		 6423 if (queue_thermal && rdev->pm.dpm_enabled)
6424 schedule_work(&rdev->pm.dpm.thermal.work);
25a857fb 6425 rdev->ih.rptr = rptr;
c20dc369
CK		 6426 atomic_set(&rdev->ih.lock, 0);
6427
6428 /* make sure wptr hasn't changed while processing */
6429 wptr = si_get_ih_wptr(rdev);
6430 if (wptr != rptr)
6431 goto restart_ih;
6432
25a857fb
AD		 6433 return IRQ_HANDLED;
6434}
6435
9b136d51
AD		 6436/*
6437 * startup/shutdown callbacks
6438 */
fa25c22e
JG		 6439static void si_uvd_init(struct radeon_device *rdev)
6440{
6441 int r;
6442
6443 if (!rdev->has_uvd)
6444 return;
6445
6446 r = radeon_uvd_init(rdev);
6447 if (r) {
6448 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
6449 /*
6450 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
6451 * to early fails uvd_v2_2_resume() and thus nothing happens
6452 * there. So it is pointless to try to go through that code
6453 * hence why we disable uvd here.
6454 */
43302df9 6455 rdev->has_uvd = false;
fa25c22e
JG		 6456 return;
6457 }
6458 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
6459 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
6460}
6461
6462static void si_uvd_start(struct radeon_device *rdev)
6463{
6464 int r;
6465
6466 if (!rdev->has_uvd)
6467 return;
6468
6469 r = uvd_v2_2_resume(rdev);
6470 if (r) {
6471 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
6472 goto error;
6473 }
6474 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
6475 if (r) {
6476 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
6477 goto error;
6478 }
6479 return;
6480
6481error:
6482 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6483}
6484
6485static void si_uvd_resume(struct radeon_device *rdev)
6486{
6487 struct radeon_ring *ring;
6488 int r;
6489
6490 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
6491 return;
6492
6493 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
70a033d2 6494 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
fa25c22e
JG		 6495 if (r) {
6496 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
6497 return;
6498 }
6499 r = uvd_v1_0_init(rdev);
6500 if (r) {
6501 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
6502 return;
6503 }
6504}
6505
d18dd759
JG		 6506static void si_vce_init(struct radeon_device *rdev)
6507{
6508 int r;
6509
6510 if (!rdev->has_vce)
6511 return;
6512
6513 r = radeon_vce_init(rdev);
6514 if (r) {
6515 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
6516 /*
6517 * At this point rdev->vce.vcpu_bo is NULL which trickles down
6518 * to early fails si_vce_start() and thus nothing happens
6519 * there. So it is pointless to try to go through that code
6520 * hence why we disable vce here.
6521 */
43302df9 6522 rdev->has_vce = false;
d18dd759
JG		 6523 return;
6524 }
6525 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
6526 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
6527 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
6528 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
6529}
6530
6531static void si_vce_start(struct radeon_device *rdev)
6532{
6533 int r;
6534
6535 if (!rdev->has_vce)
6536 return;
6537
6538 r = radeon_vce_resume(rdev);
6539 if (r) {
6540 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6541 goto error;
6542 }
6543 r = vce_v1_0_resume(rdev);
6544 if (r) {
6545 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6546 goto error;
6547 }
6548 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
6549 if (r) {
6550 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
6551 goto error;
6552 }
6553 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
6554 if (r) {
6555 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
6556 goto error;
6557 }
6558 return;
6559
6560error:
6561 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6562 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6563}
6564
6565static void si_vce_resume(struct radeon_device *rdev)
6566{
6567 struct radeon_ring *ring;
6568 int r;
6569
6570 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
6571 return;
6572
6573 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
6574 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6575 if (r) {
6576 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6577 return;
6578 }
6579 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
6580 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6581 if (r) {
6582 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6583 return;
6584 }
6585 r = vce_v1_0_init(rdev);
6586 if (r) {
6587 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
6588 return;
6589 }
6590}
6591
9b136d51
AD		 6592static int si_startup(struct radeon_device *rdev)
6593{
6594 struct radeon_ring *ring;
6595 int r;
6596
b9d305df
AD		 6597 /* enable pcie gen2/3 link */
6598 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6599 /* enable aspm */
6600 si_program_aspm(rdev);
b9d305df 6601
e5903d39
AD		 6602 /* scratch needs to be initialized before MC */
6603 r = r600_vram_scratch_init(rdev);
6604 if (r)
6605 return r;
6606
6fab3feb
AD		 6607 si_mc_program(rdev);
6608
6c7bccea
AD		 6609 if (!rdev->pm.dpm_enabled) {
6610 r = si_mc_load_microcode(rdev);
6611 if (r) {
6612 DRM_ERROR("Failed to load MC firmware!\n");
6613 return r;
6614 }
9b136d51
AD		 6615 }
6616
9b136d51
AD		 6617 r = si_pcie_gart_enable(rdev);
6618 if (r)
6619 return r;
6620 si_gpu_init(rdev);
6621
9b136d51 6622 /* allocate rlc buffers */
1fd11777
AD		 6623 if (rdev->family == CHIP_VERDE) {
6624 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6625 rdev->rlc.reg_list_size =
6626 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6627 }
6628 rdev->rlc.cs_data = si_cs_data;
6629 r = sumo_rlc_init(rdev);
9b136d51
AD		 6630 if (r) {
6631 DRM_ERROR("Failed to init rlc BOs!\n");
6632 return r;
6633 }
6634
6635 /* allocate wb buffer */
6636 r = radeon_wb_init(rdev);
6637 if (r)
6638 return r;
6639
6640 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6641 if (r) {
6642 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6643 return r;
6644 }
6645
6646 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6647 if (r) {
6648 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6649 return r;
6650 }
6651
6652 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6653 if (r) {
6654 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6655 return r;
6656 }
6657
8c5fd7ef
AD		 6658 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6659 if (r) {
6660 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6661 return r;
6662 }
6663
6664 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6665 if (r) {
6666 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6667 return r;
6668 }
6669
fa25c22e 6670 si_uvd_start(rdev);
d18dd759 6671 si_vce_start(rdev);
a918efab 6672
9b136d51 6673 /* Enable IRQ */
e49f3959
AH		 6674 if (!rdev->irq.installed) {
6675 r = radeon_irq_kms_init(rdev);
6676 if (r)
6677 return r;
6678 }
6679
9b136d51
AD		 6680 r = si_irq_init(rdev);
6681 if (r) {
6682 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6683 radeon_irq_kms_fini(rdev);
6684 return r;
6685 }
6686 si_irq_set(rdev);
6687
6688 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6689 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 6690 RADEON_CP_PACKET2);
9b136d51
AD		 6691 if (r)
6692 return r;
6693
6694 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6695 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
2e1e6dad 6696 RADEON_CP_PACKET2);
9b136d51
AD		 6697 if (r)
6698 return r;
6699
6700 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6701 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
2e1e6dad 6702 RADEON_CP_PACKET2);
9b136d51
AD		 6703 if (r)
6704 return r;
6705
8c5fd7ef
AD		 6706 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6707 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 6708 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6709 if (r)
6710 return r;
6711
6712 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6713 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 6714 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6715 if (r)
6716 return r;
6717
9b136d51
AD		 6718 r = si_cp_load_microcode(rdev);
6719 if (r)
6720 return r;
6721 r = si_cp_resume(rdev);
6722 if (r)
6723 return r;
6724
8c5fd7ef
AD		 6725 r = cayman_dma_resume(rdev);
6726 if (r)
6727 return r;
6728
fa25c22e 6729 si_uvd_resume(rdev);
d18dd759 6730 si_vce_resume(rdev);
a918efab 6731
2898c348
CK		 6732 r = radeon_ib_pool_init(rdev);
6733 if (r) {
6734 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6735 return r;
2898c348 6736 }
9b136d51 6737
c6105f24
CK		 6738 r = radeon_vm_manager_init(rdev);
6739 if (r) {
6740 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6741 return r;
c6105f24 6742 }
9b136d51 6743
bfc1f97d 6744 r = radeon_audio_init(rdev);
b530602f
AD		 6745 if (r)
6746 return r;
6747
9b136d51
AD		 6748 return 0;
6749}
6750
6751int si_resume(struct radeon_device *rdev)
6752{
6753 int r;
6754
6755 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6756 * posting will perform necessary task to bring back GPU into good
6757 * shape.
6758 */
6759 /* post card */
6760 atom_asic_init(rdev->mode_info.atom_context);
6761
205996c0
AD		 6762 /* init golden registers */
6763 si_init_golden_registers(rdev);
6764
bc6a6295
AD		 6765 if (rdev->pm.pm_method == PM_METHOD_DPM)
6766 radeon_pm_resume(rdev);
6c7bccea 6767
9b136d51
AD		 6768 rdev->accel_working = true;
6769 r = si_startup(rdev);
6770 if (r) {
6771 DRM_ERROR("si startup failed on resume\n");
6772 rdev->accel_working = false;
6773 return r;
6774 }
6775
6776 return r;
6777
6778}
6779
6780int si_suspend(struct radeon_device *rdev)
6781{
6c7bccea 6782 radeon_pm_suspend(rdev);
7991d665 6783 radeon_audio_fini(rdev);
fa3daf9a 6784 radeon_vm_manager_fini(rdev);
9b136d51 6785 si_cp_enable(rdev, false);
8c5fd7ef 6786 cayman_dma_stop(rdev);
1df0d523 6787 if (rdev->has_uvd) {
1df0d523 6788 radeon_uvd_suspend(rdev);
dfd6879b 6789 uvd_v1_0_fini(rdev);
1df0d523 6790 }
d18dd759
JG		 6791 if (rdev->has_vce)
6792 radeon_vce_suspend(rdev);
e16866ec
AD		 6793 si_fini_pg(rdev);
6794 si_fini_cg(rdev);
9b136d51
AD		 6795 si_irq_suspend(rdev);
6796 radeon_wb_disable(rdev);
6797 si_pcie_gart_disable(rdev);
6798 return 0;
6799}
6800
6801/* Plan is to move initialization in that function and use
6802 * helper function so that radeon_device_init pretty much
6803 * do nothing more than calling asic specific function. This
6804 * should also allow to remove a bunch of callback function
6805 * like vram_info.
6806 */
6807int si_init(struct radeon_device *rdev)
6808{
6809 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6810 int r;
6811
9b136d51
AD		 6812 /* Read BIOS */
6813 if (!radeon_get_bios(rdev)) {
6814 if (ASIC_IS_AVIVO(rdev))
6815 return -EINVAL;
6816 }
6817 /* Must be an ATOMBIOS */
6818 if (!rdev->is_atom_bios) {
6819 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6820 return -EINVAL;
6821 }
6822 r = radeon_atombios_init(rdev);
6823 if (r)
6824 return r;
6825
6826 /* Post card if necessary */
6827 if (!radeon_card_posted(rdev)) {
6828 if (!rdev->bios) {
6829 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6830 return -EINVAL;
6831 }
6832 DRM_INFO("GPU not posted. posting now...\n");
6833 atom_asic_init(rdev->mode_info.atom_context);
6834 }
205996c0
AD		 6835 /* init golden registers */
6836 si_init_golden_registers(rdev);
9b136d51
AD		 6837 /* Initialize scratch registers */
6838 si_scratch_init(rdev);
6839 /* Initialize surface registers */
6840 radeon_surface_init(rdev);
6841 /* Initialize clocks */
6842 radeon_get_clock_info(rdev->ddev);
6843
6844 /* Fence driver */
519424d7 6845 radeon_fence_driver_init(rdev);
9b136d51
AD		 6846
6847 /* initialize memory controller */
6848 r = si_mc_init(rdev);
6849 if (r)
6850 return r;
6851 /* Memory manager */
6852 r = radeon_bo_init(rdev);
6853 if (r)
6854 return r;
6855
01ac8794
AD		 6856 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6857 !rdev->rlc_fw || !rdev->mc_fw) {
6858 r = si_init_microcode(rdev);
6859 if (r) {
6860 DRM_ERROR("Failed to load firmware!\n");
6861 return r;
6862 }
6863 }
6864
6c7bccea
AD		 6865 /* Initialize power management */
6866 radeon_pm_init(rdev);
6867
9b136d51
AD		 6868 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6869 ring->ring_obj = NULL;
6870 r600_ring_init(rdev, ring, 1024 * 1024);
6871
6872 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6873 ring->ring_obj = NULL;
6874 r600_ring_init(rdev, ring, 1024 * 1024);
6875
6876 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6877 ring->ring_obj = NULL;
6878 r600_ring_init(rdev, ring, 1024 * 1024);
6879
8c5fd7ef
AD		 6880 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6881 ring->ring_obj = NULL;
6882 r600_ring_init(rdev, ring, 64 * 1024);
6883
6884 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6885 ring->ring_obj = NULL;
6886 r600_ring_init(rdev, ring, 64 * 1024);
6887
fa25c22e 6888 si_uvd_init(rdev);
d18dd759 6889 si_vce_init(rdev);
a918efab 6890
9b136d51
AD		 6891 rdev->ih.ring_obj = NULL;
6892 r600_ih_ring_init(rdev, 64 * 1024);
6893
6894 r = r600_pcie_gart_init(rdev);
6895 if (r)
6896 return r;
6897
9b136d51 6898 rdev->accel_working = true;
9b136d51
AD		 6899 r = si_startup(rdev);
6900 if (r) {
6901 dev_err(rdev->dev, "disabling GPU acceleration\n");
6902 si_cp_fini(rdev);
8c5fd7ef 6903 cayman_dma_fini(rdev);
9b136d51 6904 si_irq_fini(rdev);
1fd11777 6905 sumo_rlc_fini(rdev);
9b136d51 6906 radeon_wb_fini(rdev);
2898c348 6907 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6908 radeon_vm_manager_fini(rdev);
6909 radeon_irq_kms_fini(rdev);
6910 si_pcie_gart_fini(rdev);
6911 rdev->accel_working = false;
6912 }
6913
6914 /* Don't start up if the MC ucode is missing.
6915 * The default clocks and voltages before the MC ucode
6916 * is loaded are not suffient for advanced operations.
6917 */
6918 if (!rdev->mc_fw) {
6919 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6920 return -EINVAL;
6921 }
6922
6923 return 0;
6924}
6925
6926void si_fini(struct radeon_device *rdev)
6927{
6c7bccea 6928 radeon_pm_fini(rdev);
9b136d51 6929 si_cp_fini(rdev);
8c5fd7ef 6930 cayman_dma_fini(rdev);
f8f84ac5 6931 si_fini_pg(rdev);
e16866ec 6932 si_fini_cg(rdev);
9b136d51 6933 si_irq_fini(rdev);
1fd11777 6934 sumo_rlc_fini(rdev);
9b136d51
AD		 6935 radeon_wb_fini(rdev);
6936 radeon_vm_manager_fini(rdev);
2898c348 6937 radeon_ib_pool_fini(rdev);
9b136d51 6938 radeon_irq_kms_fini(rdev);
2858c00d 6939 if (rdev->has_uvd) {
e409b128 6940 uvd_v1_0_fini(rdev);
1df0d523 6941 radeon_uvd_fini(rdev);
2858c00d 6942 }
d18dd759
JG		 6943 if (rdev->has_vce)
6944 radeon_vce_fini(rdev);
9b136d51
AD		 6945 si_pcie_gart_fini(rdev);
6946 r600_vram_scratch_fini(rdev);
6947 radeon_gem_fini(rdev);
9b136d51
AD		 6948 radeon_fence_driver_fini(rdev);
6949 radeon_bo_fini(rdev);
6950 radeon_atombios_fini(rdev);
6951 kfree(rdev->bios);
6952 rdev->bios = NULL;
6953}
6954
6759a0a7 6955/**
d0418894 6956 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6957 *
6958 * @rdev: radeon_device pointer
6959 *
6960 * Fetches a GPU clock counter snapshot (SI).
6961 * Returns the 64 bit clock counter snapshot.
6962 */
d0418894 6963uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6964{
6965 uint64_t clock;
6966
6967 mutex_lock(&rdev->gpu_clock_mutex);
6968 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6969 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
3cf8bb1a 6970 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6759a0a7
MO		 6971 mutex_unlock(&rdev->gpu_clock_mutex);
6972 return clock;
6973}
2539eb02 6974
2539eb02
CK		 6975int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6976{
facd112d 6977 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6978 int r;
6979
4ed10835
CK		 6980 /* bypass vclk and dclk with bclk */
6981 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6982 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6983 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6984
6985 /* put PLL in bypass mode */
6986 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6987
6988 if (!vclk || !dclk) {
a17d4996 6989 /* keep the Bypass mode */
4ed10835
CK		 6990 return 0;
6991 }
6992
facd112d
CK		 6993 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6994 16384, 0x03FFFFFF, 0, 128, 5,
6995 &fb_div, &vclk_div, &dclk_div);
6996 if (r)
6997 return r;
2539eb02
CK		 6998
6999 /* set RESET_ANTI_MUX to 0 */
7000 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7001
7002 /* set VCO_MODE to 1 */
7003 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7004
a17d4996 7005 /* disable sleep mode */
2539eb02
CK		 7006 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7007
7008 /* deassert UPLL_RESET */
7009 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7010
7011 mdelay(1);
7012
facd112d 7013 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 7014 if (r)
7015 return r;
7016
7017 /* assert UPLL_RESET again */
7018 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7019
7020 /* disable spread spectrum. */
7021 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7022
7023 /* set feedback divider */
facd112d 7024 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 7025
7026 /* set ref divider to 0 */
7027 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7028
facd112d 7029 if (fb_div < 307200)
2539eb02
CK		 7030 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7031 else
7032 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7033
7034 /* set PDIV_A and PDIV_B */
7035 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 7036 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 7037 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7038
7039 /* give the PLL some time to settle */
7040 mdelay(15);
7041
7042 /* deassert PLL_RESET */
7043 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7044
7045 mdelay(15);
7046
7047 /* switch from bypass mode to normal mode */
7048 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7049
facd112d 7050 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 7051 if (r)
7052 return r;
7053
7054 /* switch VCLK and DCLK selection */
7055 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7056 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7057 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7058
7059 mdelay(100);
7060
7061 return 0;
7062}
b9d305df
AD		 7063
7064static void si_pcie_gen3_enable(struct radeon_device *rdev)
7065{
7066 struct pci_dev *root = rdev->pdev->bus->self;
5f152a57 7067 enum pci_bus_speed speed_cap;
5f152a57
AD		 7068 u32 speed_cntl, current_data_rate;
7069 int i;
b9d305df
AD		 7070 u16 tmp16;
7071
0bd252de
AW		 7072 if (pci_is_root_bus(rdev->pdev->bus))
7073 return;
7074
b9d305df
AD		 7075 if (radeon_pcie_gen2 == 0)
7076 return;
7077
7078 if (rdev->flags & RADEON_IS_IGP)
7079 return;
7080
7081 if (!(rdev->flags & RADEON_IS_PCIE))
7082 return;
7083
5f152a57
AD		 7084 speed_cap = pcie_get_speed_cap(root);
7085 if (speed_cap == PCI_SPEED_UNKNOWN)
b9d305df
AD		 7086 return;
7087
5f152a57
AD		 7088 if ((speed_cap != PCIE_SPEED_8_0GT) &&
7089 (speed_cap != PCIE_SPEED_5_0GT))
b9d305df
AD		 7090 return;
7091
7092 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7093 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7094 LC_CURRENT_DATA_RATE_SHIFT;
5f152a57 7095 if (speed_cap == PCIE_SPEED_8_0GT) {
b9d305df
AD		 7096 if (current_data_rate == 2) {
7097 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7098 return;
7099 }
7100 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
5f152a57 7101 } else if (speed_cap == PCIE_SPEED_5_0GT) {
b9d305df
AD		 7102 if (current_data_rate == 1) {
7103 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7104 return;
7105 }
7106 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7107 }
7108
3d581b11 7109 if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev))
b9d305df
AD		 7110 return;
7111
5f152a57 7112 if (speed_cap == PCIE_SPEED_8_0GT) {
b9d305df
AD		 7113 /* re-try equalization if gen3 is not already enabled */
7114 if (current_data_rate != 2) {
7115 u16 bridge_cfg, gpu_cfg;
7116 u16 bridge_cfg2, gpu_cfg2;
7117 u32 max_lw, current_lw, tmp;
7118
7189576e
IJ		 7119 pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
7120 pcie_capability_set_word(rdev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
b9d305df
AD		 7121
7122 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7123 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7124 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7125
7126 if (current_lw < max_lw) {
7127 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7128 if (tmp & LC_RENEGOTIATION_SUPPORT) {
7129 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7130 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7131 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7132 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7133 }
7134 }
7135
7136 for (i = 0; i < 10; i++) {
7137 /* check status */
3d581b11
FL		 7138 pcie_capability_read_word(rdev->pdev,
7139 PCI_EXP_DEVSTA,
7140 &tmp16);
b9d305df
AD		 7141 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7142 break;
7143
3d581b11
FL		 7144 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
7145 &bridge_cfg);
7146 pcie_capability_read_word(rdev->pdev,
7147 PCI_EXP_LNKCTL,
7148 &gpu_cfg);
b9d305df 7149
3d581b11
FL		 7150 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
7151 &bridge_cfg2);
7152 pcie_capability_read_word(rdev->pdev,
7153 PCI_EXP_LNKCTL2,
7154 &gpu_cfg2);
b9d305df
AD		 7155
7156 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7157 tmp |= LC_SET_QUIESCE;
7158 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7159
7160 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7161 tmp |= LC_REDO_EQ;
7162 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7163
2f2debb5 7164 msleep(100);
b9d305df
AD		 7165
7166 /* linkctl */
7189576e
IJ		 7167 pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
7168 PCI_EXP_LNKCTL_HAWD,
7169 bridge_cfg &
7170 PCI_EXP_LNKCTL_HAWD);
7171 pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL,
7172 PCI_EXP_LNKCTL_HAWD,
7173 gpu_cfg &
7174 PCI_EXP_LNKCTL_HAWD);
b9d305df
AD		 7175
7176 /* linkctl2 */
756762ae
IJ		 7177 pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL2,
7178 PCI_EXP_LNKCTL2_ENTER_COMP |
7179 PCI_EXP_LNKCTL2_TX_MARGIN,
7180 bridge_cfg2 &
7181 (PCI_EXP_LNKCTL2_ENTER_COMP |
7182 PCI_EXP_LNKCTL2_TX_MARGIN));
7183 pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL2,
7184 PCI_EXP_LNKCTL2_ENTER_COMP |
7185 PCI_EXP_LNKCTL2_TX_MARGIN,
7186 gpu_cfg2 &
7187 (PCI_EXP_LNKCTL2_ENTER_COMP |
7188 PCI_EXP_LNKCTL2_TX_MARGIN));
b9d305df
AD		 7189
7190 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7191 tmp &= ~LC_SET_QUIESCE;
7192 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7193 }
7194 }
7195 }
7196
7197 /* set the link speed */
7198 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7199 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7200 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7201
756762ae 7202 tmp16 = 0;
5f152a57 7203 if (speed_cap == PCIE_SPEED_8_0GT)
ca56f99c 7204 tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
5f152a57 7205 else if (speed_cap == PCIE_SPEED_5_0GT)
ca56f99c 7206 tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
b9d305df 7207 else
ca56f99c 7208 tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
756762ae
IJ		 7209 pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL2,
7210 PCI_EXP_LNKCTL2_TLS, tmp16);
b9d305df
AD		 7211
7212 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7213 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7214 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7215
7216 for (i = 0; i < rdev->usec_timeout; i++) {
7217 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7218 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7219 break;
7220 udelay(1);
7221 }
7222}
7223
e0bcf165
AD		 7224static void si_program_aspm(struct radeon_device *rdev)
7225{
7226 u32 data, orig;
7227 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7228 bool disable_clkreq = false;
7229
1294d4a3
AD		 7230 if (radeon_aspm == 0)
7231 return;
7232
e0bcf165
AD		 7233 if (!(rdev->flags & RADEON_IS_PCIE))
7234 return;
7235
7236 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7237 data &= ~LC_XMIT_N_FTS_MASK;
7238 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7239 if (orig != data)
7240 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7241
7242 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7243 data |= LC_GO_TO_RECOVERY;
7244 if (orig != data)
7245 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7246
7247 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7248 data |= P_IGNORE_EDB_ERR;
7249 if (orig != data)
7250 WREG32_PCIE(PCIE_P_CNTL, data);
7251
7252 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7253 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7254 data |= LC_PMI_TO_L1_DIS;
7255 if (!disable_l0s)
7256 data |= LC_L0S_INACTIVITY(7);
7257
7258 if (!disable_l1) {
7259 data |= LC_L1_INACTIVITY(7);
7260 data &= ~LC_PMI_TO_L1_DIS;
7261 if (orig != data)
7262 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7263
7264 if (!disable_plloff_in_l1) {
7265 bool clk_req_support;
7266
7267 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7268 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7269 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7270 if (orig != data)
7271 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7272
7273 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7274 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7275 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7276 if (orig != data)
7277 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7278
7279 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7280 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7281 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7282 if (orig != data)
7283 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7284
7285 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7286 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7287 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7288 if (orig != data)
7289 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7290
7291 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7292 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7293 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7294 if (orig != data)
7295 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7296
7297 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7298 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7299 if (orig != data)
7300 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7301
7302 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7303 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7304 if (orig != data)
7305 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7306
7307 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7308 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7309 if (orig != data)
7310 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7311
7312 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7313 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7314 if (orig != data)
7315 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7316
7317 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7318 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7319 if (orig != data)
7320 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7321
7322 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7323 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7324 if (orig != data)
7325 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7326
7327 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7328 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7329 if (orig != data)
7330 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7331 }
7332 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7333 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7334 data |= LC_DYN_LANES_PWR_STATE(3);
7335 if (orig != data)
7336 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7337
7338 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7339 data &= ~LS2_EXIT_TIME_MASK;
7340 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7341 data |= LS2_EXIT_TIME(5);
7342 if (orig != data)
7343 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7344
7345 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7346 data &= ~LS2_EXIT_TIME_MASK;
7347 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7348 data |= LS2_EXIT_TIME(5);
7349 if (orig != data)
7350 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7351
0bd252de
AW		 7352 if (!disable_clkreq &&
7353 !pci_is_root_bus(rdev->pdev->bus)) {
e0bcf165
AD		 7354 struct pci_dev *root = rdev->pdev->bus->self;
7355 u32 lnkcap;
7356
7357 clk_req_support = false;
7358 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7359 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7360 clk_req_support = true;
7361 } else {
7362 clk_req_support = false;
7363 }
7364
7365 if (clk_req_support) {
7366 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7367 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7368 if (orig != data)
7369 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7370
7371 orig = data = RREG32(THM_CLK_CNTL);
7372 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7373 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7374 if (orig != data)
7375 WREG32(THM_CLK_CNTL, data);
7376
7377 orig = data = RREG32(MISC_CLK_CNTL);
7378 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7379 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7380 if (orig != data)
7381 WREG32(MISC_CLK_CNTL, data);
7382
7383 orig = data = RREG32(CG_CLKPIN_CNTL);
7384 data &= ~BCLK_AS_XCLK;
7385 if (orig != data)
7386 WREG32(CG_CLKPIN_CNTL, data);
7387
7388 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7389 data &= ~FORCE_BIF_REFCLK_EN;
7390 if (orig != data)
7391 WREG32(CG_CLKPIN_CNTL_2, data);
7392
7393 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7394 data &= ~MPLL_CLKOUT_SEL_MASK;
7395 data |= MPLL_CLKOUT_SEL(4);
7396 if (orig != data)
7397 WREG32(MPLL_BYPASSCLK_SEL, data);
7398
7399 orig = data = RREG32(SPLL_CNTL_MODE);
7400 data &= ~SPLL_REFCLK_SEL_MASK;
7401 if (orig != data)
7402 WREG32(SPLL_CNTL_MODE, data);
7403 }
7404 }
7405 } else {
7406 if (orig != data)
7407 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7408 }
7409
7410 orig = data = RREG32_PCIE(PCIE_CNTL2);
7411 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7412 if (orig != data)
7413 WREG32_PCIE(PCIE_CNTL2, data);
7414
7415 if (!disable_l0s) {
7416 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7417 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7418 data = RREG32_PCIE(PCIE_LC_STATUS1);
7419 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7420 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7421 data &= ~LC_L0S_INACTIVITY_MASK;
7422 if (orig != data)
7423 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7424 }
7425 }
7426 }
7427}
b7af630c 7428
22e5808e 7429static int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
b7af630c 7430{
3cf8bb1a 7431 unsigned i;
b7af630c 7432
3cf8bb1a
JG		 7433 /* make sure VCEPLL_CTLREQ is deasserted */
7434 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
b7af630c 7435
3cf8bb1a 7436 mdelay(10);
b7af630c 7437
3cf8bb1a
JG		 7438 /* assert UPLL_CTLREQ */
7439 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
b7af630c 7440
3cf8bb1a
JG		 7441 /* wait for CTLACK and CTLACK2 to get asserted */
7442 for (i = 0; i < 100; ++i) {
7443 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7444 if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7445 break;
7446 mdelay(10);
7447 }
b7af630c 7448
3cf8bb1a
JG		 7449 /* deassert UPLL_CTLREQ */
7450 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
b7af630c 7451
3cf8bb1a
JG		 7452 if (i == 100) {
7453 DRM_ERROR("Timeout setting UVD clocks!\n");
7454 return -ETIMEDOUT;
7455 }
b7af630c 7456
3cf8bb1a 7457 return 0;
b7af630c
CK		 7458}
7459
7460int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7461{
7462 unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7463 int r;
7464
7465 /* bypass evclk and ecclk with bclk */
7466 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7467 EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7468 ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7469
7470 /* put PLL in bypass mode */
7471 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7472 ~VCEPLL_BYPASS_EN_MASK);
7473
7474 if (!evclk || !ecclk) {
7475 /* keep the Bypass mode, put PLL to sleep */
7476 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7477 ~VCEPLL_SLEEP_MASK);
7478 return 0;
7479 }
7480
7481 r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7482 16384, 0x03FFFFFF, 0, 128, 5,
7483 &fb_div, &evclk_div, &ecclk_div);
7484 if (r)
7485 return r;
7486
7487 /* set RESET_ANTI_MUX to 0 */
7488 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7489
7490 /* set VCO_MODE to 1 */
7491 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7492 ~VCEPLL_VCO_MODE_MASK);
7493
7494 /* toggle VCEPLL_SLEEP to 1 then back to 0 */
7495 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7496 ~VCEPLL_SLEEP_MASK);
7497 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7498
7499 /* deassert VCEPLL_RESET */
7500 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7501
7502 mdelay(1);
7503
7504 r = si_vce_send_vcepll_ctlreq(rdev);
7505 if (r)
7506 return r;
7507
7508 /* assert VCEPLL_RESET again */
7509 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7510
7511 /* disable spread spectrum. */
7512 WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7513
7514 /* set feedback divider */
7515 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7516
7517 /* set ref divider to 0 */
7518 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7519
7520 /* set PDIV_A and PDIV_B */
7521 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7522 VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7523 ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7524
7525 /* give the PLL some time to settle */
7526 mdelay(15);
7527
7528 /* deassert PLL_RESET */
7529 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7530
7531 mdelay(15);
7532
7533 /* switch from bypass mode to normal mode */
7534 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7535
7536 r = si_vce_send_vcepll_ctlreq(rdev);
7537 if (r)
7538 return r;
7539
7540 /* switch VCLK and DCLK selection */
7541 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7542 EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7543 ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7544
7545 mdelay(100);
7546
7547 return 0;
7548}
Linux 6.x block layer and io_uring tree(s)