projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/dpm: save some display parameters for DPM
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c
AD		 24#include <linux/firmware.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include <linux/module.h>
760285e7 28#include <drm/drmP.h>
43b3cd99
AD		 29#include "radeon.h"
30#include "radeon_asic.h"
760285e7 31#include <drm/radeon_drm.h>
43b3cd99
AD		 32#include "sid.h"
33#include "atom.h"
48c0c902 34#include "si_blit_shaders.h"
bd8cd539 35#include "clearstate_si.h"
43b3cd99 36
0f0de06c
AD		 37#define SI_PFP_UCODE_SIZE 2144
38#define SI_PM4_UCODE_SIZE 2144
39#define SI_CE_UCODE_SIZE 2144
40#define SI_RLC_UCODE_SIZE 2048
41#define SI_MC_UCODE_SIZE 7769
bcc7f5d2 42#define OLAND_MC_UCODE_SIZE 7863
0f0de06c
AD		 43
44MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
45MODULE_FIRMWARE("radeon/TAHITI_me.bin");
46MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
47MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
48MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
49MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
50MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
51MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
52MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
53MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
54MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
55MODULE_FIRMWARE("radeon/VERDE_me.bin");
56MODULE_FIRMWARE("radeon/VERDE_ce.bin");
57MODULE_FIRMWARE("radeon/VERDE_mc.bin");
58MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
bcc7f5d2
AD		 59MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
60MODULE_FIRMWARE("radeon/OLAND_me.bin");
61MODULE_FIRMWARE("radeon/OLAND_ce.bin");
62MODULE_FIRMWARE("radeon/OLAND_mc.bin");
63MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
c04c00b4
AD		 64MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
65MODULE_FIRMWARE("radeon/HAINAN_me.bin");
66MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
67MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
68MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
0f0de06c 69
b9d305df 70static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 71static void si_program_aspm(struct radeon_device *rdev);
25a857fb
AD		 72extern int r600_ih_ring_alloc(struct radeon_device *rdev);
73extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 74extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 75extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
76extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 77extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 78extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 79extern bool evergreen_is_display_hung(struct radeon_device *rdev);
0a96d72b 80
6d8cf000
AD		 81static const u32 verde_rlc_save_restore_register_list[] =
82{
83 (0x8000 << 16) | (0x98f4 >> 2),
84 0x00000000,
85 (0x8040 << 16) | (0x98f4 >> 2),
86 0x00000000,
87 (0x8000 << 16) | (0xe80 >> 2),
88 0x00000000,
89 (0x8040 << 16) | (0xe80 >> 2),
90 0x00000000,
91 (0x8000 << 16) | (0x89bc >> 2),
92 0x00000000,
93 (0x8040 << 16) | (0x89bc >> 2),
94 0x00000000,
95 (0x8000 << 16) | (0x8c1c >> 2),
96 0x00000000,
97 (0x8040 << 16) | (0x8c1c >> 2),
98 0x00000000,
99 (0x9c00 << 16) | (0x98f0 >> 2),
100 0x00000000,
101 (0x9c00 << 16) | (0xe7c >> 2),
102 0x00000000,
103 (0x8000 << 16) | (0x9148 >> 2),
104 0x00000000,
105 (0x8040 << 16) | (0x9148 >> 2),
106 0x00000000,
107 (0x9c00 << 16) | (0x9150 >> 2),
108 0x00000000,
109 (0x9c00 << 16) | (0x897c >> 2),
110 0x00000000,
111 (0x9c00 << 16) | (0x8d8c >> 2),
112 0x00000000,
113 (0x9c00 << 16) | (0xac54 >> 2),
114 0X00000000,
115 0x3,
116 (0x9c00 << 16) | (0x98f8 >> 2),
117 0x00000000,
118 (0x9c00 << 16) | (0x9910 >> 2),
119 0x00000000,
120 (0x9c00 << 16) | (0x9914 >> 2),
121 0x00000000,
122 (0x9c00 << 16) | (0x9918 >> 2),
123 0x00000000,
124 (0x9c00 << 16) | (0x991c >> 2),
125 0x00000000,
126 (0x9c00 << 16) | (0x9920 >> 2),
127 0x00000000,
128 (0x9c00 << 16) | (0x9924 >> 2),
129 0x00000000,
130 (0x9c00 << 16) | (0x9928 >> 2),
131 0x00000000,
132 (0x9c00 << 16) | (0x992c >> 2),
133 0x00000000,
134 (0x9c00 << 16) | (0x9930 >> 2),
135 0x00000000,
136 (0x9c00 << 16) | (0x9934 >> 2),
137 0x00000000,
138 (0x9c00 << 16) | (0x9938 >> 2),
139 0x00000000,
140 (0x9c00 << 16) | (0x993c >> 2),
141 0x00000000,
142 (0x9c00 << 16) | (0x9940 >> 2),
143 0x00000000,
144 (0x9c00 << 16) | (0x9944 >> 2),
145 0x00000000,
146 (0x9c00 << 16) | (0x9948 >> 2),
147 0x00000000,
148 (0x9c00 << 16) | (0x994c >> 2),
149 0x00000000,
150 (0x9c00 << 16) | (0x9950 >> 2),
151 0x00000000,
152 (0x9c00 << 16) | (0x9954 >> 2),
153 0x00000000,
154 (0x9c00 << 16) | (0x9958 >> 2),
155 0x00000000,
156 (0x9c00 << 16) | (0x995c >> 2),
157 0x00000000,
158 (0x9c00 << 16) | (0x9960 >> 2),
159 0x00000000,
160 (0x9c00 << 16) | (0x9964 >> 2),
161 0x00000000,
162 (0x9c00 << 16) | (0x9968 >> 2),
163 0x00000000,
164 (0x9c00 << 16) | (0x996c >> 2),
165 0x00000000,
166 (0x9c00 << 16) | (0x9970 >> 2),
167 0x00000000,
168 (0x9c00 << 16) | (0x9974 >> 2),
169 0x00000000,
170 (0x9c00 << 16) | (0x9978 >> 2),
171 0x00000000,
172 (0x9c00 << 16) | (0x997c >> 2),
173 0x00000000,
174 (0x9c00 << 16) | (0x9980 >> 2),
175 0x00000000,
176 (0x9c00 << 16) | (0x9984 >> 2),
177 0x00000000,
178 (0x9c00 << 16) | (0x9988 >> 2),
179 0x00000000,
180 (0x9c00 << 16) | (0x998c >> 2),
181 0x00000000,
182 (0x9c00 << 16) | (0x8c00 >> 2),
183 0x00000000,
184 (0x9c00 << 16) | (0x8c14 >> 2),
185 0x00000000,
186 (0x9c00 << 16) | (0x8c04 >> 2),
187 0x00000000,
188 (0x9c00 << 16) | (0x8c08 >> 2),
189 0x00000000,
190 (0x8000 << 16) | (0x9b7c >> 2),
191 0x00000000,
192 (0x8040 << 16) | (0x9b7c >> 2),
193 0x00000000,
194 (0x8000 << 16) | (0xe84 >> 2),
195 0x00000000,
196 (0x8040 << 16) | (0xe84 >> 2),
197 0x00000000,
198 (0x8000 << 16) | (0x89c0 >> 2),
199 0x00000000,
200 (0x8040 << 16) | (0x89c0 >> 2),
201 0x00000000,
202 (0x8000 << 16) | (0x914c >> 2),
203 0x00000000,
204 (0x8040 << 16) | (0x914c >> 2),
205 0x00000000,
206 (0x8000 << 16) | (0x8c20 >> 2),
207 0x00000000,
208 (0x8040 << 16) | (0x8c20 >> 2),
209 0x00000000,
210 (0x8000 << 16) | (0x9354 >> 2),
211 0x00000000,
212 (0x8040 << 16) | (0x9354 >> 2),
213 0x00000000,
214 (0x9c00 << 16) | (0x9060 >> 2),
215 0x00000000,
216 (0x9c00 << 16) | (0x9364 >> 2),
217 0x00000000,
218 (0x9c00 << 16) | (0x9100 >> 2),
219 0x00000000,
220 (0x9c00 << 16) | (0x913c >> 2),
221 0x00000000,
222 (0x8000 << 16) | (0x90e0 >> 2),
223 0x00000000,
224 (0x8000 << 16) | (0x90e4 >> 2),
225 0x00000000,
226 (0x8000 << 16) | (0x90e8 >> 2),
227 0x00000000,
228 (0x8040 << 16) | (0x90e0 >> 2),
229 0x00000000,
230 (0x8040 << 16) | (0x90e4 >> 2),
231 0x00000000,
232 (0x8040 << 16) | (0x90e8 >> 2),
233 0x00000000,
234 (0x9c00 << 16) | (0x8bcc >> 2),
235 0x00000000,
236 (0x9c00 << 16) | (0x8b24 >> 2),
237 0x00000000,
238 (0x9c00 << 16) | (0x88c4 >> 2),
239 0x00000000,
240 (0x9c00 << 16) | (0x8e50 >> 2),
241 0x00000000,
242 (0x9c00 << 16) | (0x8c0c >> 2),
243 0x00000000,
244 (0x9c00 << 16) | (0x8e58 >> 2),
245 0x00000000,
246 (0x9c00 << 16) | (0x8e5c >> 2),
247 0x00000000,
248 (0x9c00 << 16) | (0x9508 >> 2),
249 0x00000000,
250 (0x9c00 << 16) | (0x950c >> 2),
251 0x00000000,
252 (0x9c00 << 16) | (0x9494 >> 2),
253 0x00000000,
254 (0x9c00 << 16) | (0xac0c >> 2),
255 0x00000000,
256 (0x9c00 << 16) | (0xac10 >> 2),
257 0x00000000,
258 (0x9c00 << 16) | (0xac14 >> 2),
259 0x00000000,
260 (0x9c00 << 16) | (0xae00 >> 2),
261 0x00000000,
262 (0x9c00 << 16) | (0xac08 >> 2),
263 0x00000000,
264 (0x9c00 << 16) | (0x88d4 >> 2),
265 0x00000000,
266 (0x9c00 << 16) | (0x88c8 >> 2),
267 0x00000000,
268 (0x9c00 << 16) | (0x88cc >> 2),
269 0x00000000,
270 (0x9c00 << 16) | (0x89b0 >> 2),
271 0x00000000,
272 (0x9c00 << 16) | (0x8b10 >> 2),
273 0x00000000,
274 (0x9c00 << 16) | (0x8a14 >> 2),
275 0x00000000,
276 (0x9c00 << 16) | (0x9830 >> 2),
277 0x00000000,
278 (0x9c00 << 16) | (0x9834 >> 2),
279 0x00000000,
280 (0x9c00 << 16) | (0x9838 >> 2),
281 0x00000000,
282 (0x9c00 << 16) | (0x9a10 >> 2),
283 0x00000000,
284 (0x8000 << 16) | (0x9870 >> 2),
285 0x00000000,
286 (0x8000 << 16) | (0x9874 >> 2),
287 0x00000000,
288 (0x8001 << 16) | (0x9870 >> 2),
289 0x00000000,
290 (0x8001 << 16) | (0x9874 >> 2),
291 0x00000000,
292 (0x8040 << 16) | (0x9870 >> 2),
293 0x00000000,
294 (0x8040 << 16) | (0x9874 >> 2),
295 0x00000000,
296 (0x8041 << 16) | (0x9870 >> 2),
297 0x00000000,
298 (0x8041 << 16) | (0x9874 >> 2),
299 0x00000000,
300 0x00000000
301};
302
205996c0
AD		 303static const u32 tahiti_golden_rlc_registers[] =
304{
305 0xc424, 0xffffffff, 0x00601005,
306 0xc47c, 0xffffffff, 0x10104040,
307 0xc488, 0xffffffff, 0x0100000a,
308 0xc314, 0xffffffff, 0x00000800,
309 0xc30c, 0xffffffff, 0x800000f4,
310 0xf4a8, 0xffffffff, 0x00000000
311};
312
313static const u32 tahiti_golden_registers[] =
314{
315 0x9a10, 0x00010000, 0x00018208,
316 0x9830, 0xffffffff, 0x00000000,
317 0x9834, 0xf00fffff, 0x00000400,
318 0x9838, 0x0002021c, 0x00020200,
319 0xc78, 0x00000080, 0x00000000,
320 0xd030, 0x000300c0, 0x00800040,
321 0xd830, 0x000300c0, 0x00800040,
322 0x5bb0, 0x000000f0, 0x00000070,
323 0x5bc0, 0x00200000, 0x50100000,
324 0x7030, 0x31000311, 0x00000011,
325 0x277c, 0x00000003, 0x000007ff,
326 0x240c, 0x000007ff, 0x00000000,
327 0x8a14, 0xf000001f, 0x00000007,
328 0x8b24, 0xffffffff, 0x00ffffff,
329 0x8b10, 0x0000ff0f, 0x00000000,
330 0x28a4c, 0x07ffffff, 0x4e000000,
331 0x28350, 0x3f3f3fff, 0x2a00126a,
332 0x30, 0x000000ff, 0x0040,
333 0x34, 0x00000040, 0x00004040,
334 0x9100, 0x07ffffff, 0x03000000,
335 0x8e88, 0x01ff1f3f, 0x00000000,
336 0x8e84, 0x01ff1f3f, 0x00000000,
337 0x9060, 0x0000007f, 0x00000020,
338 0x9508, 0x00010000, 0x00010000,
339 0xac14, 0x00000200, 0x000002fb,
340 0xac10, 0xffffffff, 0x0000543b,
341 0xac0c, 0xffffffff, 0xa9210876,
342 0x88d0, 0xffffffff, 0x000fff40,
343 0x88d4, 0x0000001f, 0x00000010,
344 0x1410, 0x20000000, 0x20fffed8,
345 0x15c0, 0x000c0fc0, 0x000c0400
346};
347
348static const u32 tahiti_golden_registers2[] =
349{
350 0xc64, 0x00000001, 0x00000001
351};
352
353static const u32 pitcairn_golden_rlc_registers[] =
354{
355 0xc424, 0xffffffff, 0x00601004,
356 0xc47c, 0xffffffff, 0x10102020,
357 0xc488, 0xffffffff, 0x01000020,
358 0xc314, 0xffffffff, 0x00000800,
359 0xc30c, 0xffffffff, 0x800000a4
360};
361
362static const u32 pitcairn_golden_registers[] =
363{
364 0x9a10, 0x00010000, 0x00018208,
365 0x9830, 0xffffffff, 0x00000000,
366 0x9834, 0xf00fffff, 0x00000400,
367 0x9838, 0x0002021c, 0x00020200,
368 0xc78, 0x00000080, 0x00000000,
369 0xd030, 0x000300c0, 0x00800040,
370 0xd830, 0x000300c0, 0x00800040,
371 0x5bb0, 0x000000f0, 0x00000070,
372 0x5bc0, 0x00200000, 0x50100000,
373 0x7030, 0x31000311, 0x00000011,
374 0x2ae4, 0x00073ffe, 0x000022a2,
375 0x240c, 0x000007ff, 0x00000000,
376 0x8a14, 0xf000001f, 0x00000007,
377 0x8b24, 0xffffffff, 0x00ffffff,
378 0x8b10, 0x0000ff0f, 0x00000000,
379 0x28a4c, 0x07ffffff, 0x4e000000,
380 0x28350, 0x3f3f3fff, 0x2a00126a,
381 0x30, 0x000000ff, 0x0040,
382 0x34, 0x00000040, 0x00004040,
383 0x9100, 0x07ffffff, 0x03000000,
384 0x9060, 0x0000007f, 0x00000020,
385 0x9508, 0x00010000, 0x00010000,
386 0xac14, 0x000003ff, 0x000000f7,
387 0xac10, 0xffffffff, 0x00000000,
388 0xac0c, 0xffffffff, 0x32761054,
389 0x88d4, 0x0000001f, 0x00000010,
390 0x15c0, 0x000c0fc0, 0x000c0400
391};
392
393static const u32 verde_golden_rlc_registers[] =
394{
395 0xc424, 0xffffffff, 0x033f1005,
396 0xc47c, 0xffffffff, 0x10808020,
397 0xc488, 0xffffffff, 0x00800008,
398 0xc314, 0xffffffff, 0x00001000,
399 0xc30c, 0xffffffff, 0x80010014
400};
401
402static const u32 verde_golden_registers[] =
403{
404 0x9a10, 0x00010000, 0x00018208,
405 0x9830, 0xffffffff, 0x00000000,
406 0x9834, 0xf00fffff, 0x00000400,
407 0x9838, 0x0002021c, 0x00020200,
408 0xc78, 0x00000080, 0x00000000,
409 0xd030, 0x000300c0, 0x00800040,
410 0xd030, 0x000300c0, 0x00800040,
411 0xd830, 0x000300c0, 0x00800040,
412 0xd830, 0x000300c0, 0x00800040,
413 0x5bb0, 0x000000f0, 0x00000070,
414 0x5bc0, 0x00200000, 0x50100000,
415 0x7030, 0x31000311, 0x00000011,
416 0x2ae4, 0x00073ffe, 0x000022a2,
417 0x2ae4, 0x00073ffe, 0x000022a2,
418 0x2ae4, 0x00073ffe, 0x000022a2,
419 0x240c, 0x000007ff, 0x00000000,
420 0x240c, 0x000007ff, 0x00000000,
421 0x240c, 0x000007ff, 0x00000000,
422 0x8a14, 0xf000001f, 0x00000007,
423 0x8a14, 0xf000001f, 0x00000007,
424 0x8a14, 0xf000001f, 0x00000007,
425 0x8b24, 0xffffffff, 0x00ffffff,
426 0x8b10, 0x0000ff0f, 0x00000000,
427 0x28a4c, 0x07ffffff, 0x4e000000,
428 0x28350, 0x3f3f3fff, 0x0000124a,
429 0x28350, 0x3f3f3fff, 0x0000124a,
430 0x28350, 0x3f3f3fff, 0x0000124a,
431 0x30, 0x000000ff, 0x0040,
432 0x34, 0x00000040, 0x00004040,
433 0x9100, 0x07ffffff, 0x03000000,
434 0x9100, 0x07ffffff, 0x03000000,
435 0x8e88, 0x01ff1f3f, 0x00000000,
436 0x8e88, 0x01ff1f3f, 0x00000000,
437 0x8e88, 0x01ff1f3f, 0x00000000,
438 0x8e84, 0x01ff1f3f, 0x00000000,
439 0x8e84, 0x01ff1f3f, 0x00000000,
440 0x8e84, 0x01ff1f3f, 0x00000000,
441 0x9060, 0x0000007f, 0x00000020,
442 0x9508, 0x00010000, 0x00010000,
443 0xac14, 0x000003ff, 0x00000003,
444 0xac14, 0x000003ff, 0x00000003,
445 0xac14, 0x000003ff, 0x00000003,
446 0xac10, 0xffffffff, 0x00000000,
447 0xac10, 0xffffffff, 0x00000000,
448 0xac10, 0xffffffff, 0x00000000,
449 0xac0c, 0xffffffff, 0x00001032,
450 0xac0c, 0xffffffff, 0x00001032,
451 0xac0c, 0xffffffff, 0x00001032,
452 0x88d4, 0x0000001f, 0x00000010,
453 0x88d4, 0x0000001f, 0x00000010,
454 0x88d4, 0x0000001f, 0x00000010,
455 0x15c0, 0x000c0fc0, 0x000c0400
456};
457
458static const u32 oland_golden_rlc_registers[] =
459{
460 0xc424, 0xffffffff, 0x00601005,
461 0xc47c, 0xffffffff, 0x10104040,
462 0xc488, 0xffffffff, 0x0100000a,
463 0xc314, 0xffffffff, 0x00000800,
464 0xc30c, 0xffffffff, 0x800000f4
465};
466
467static const u32 oland_golden_registers[] =
468{
469 0x9a10, 0x00010000, 0x00018208,
470 0x9830, 0xffffffff, 0x00000000,
471 0x9834, 0xf00fffff, 0x00000400,
472 0x9838, 0x0002021c, 0x00020200,
473 0xc78, 0x00000080, 0x00000000,
474 0xd030, 0x000300c0, 0x00800040,
475 0xd830, 0x000300c0, 0x00800040,
476 0x5bb0, 0x000000f0, 0x00000070,
477 0x5bc0, 0x00200000, 0x50100000,
478 0x7030, 0x31000311, 0x00000011,
479 0x2ae4, 0x00073ffe, 0x000022a2,
480 0x240c, 0x000007ff, 0x00000000,
481 0x8a14, 0xf000001f, 0x00000007,
482 0x8b24, 0xffffffff, 0x00ffffff,
483 0x8b10, 0x0000ff0f, 0x00000000,
484 0x28a4c, 0x07ffffff, 0x4e000000,
485 0x28350, 0x3f3f3fff, 0x00000082,
486 0x30, 0x000000ff, 0x0040,
487 0x34, 0x00000040, 0x00004040,
488 0x9100, 0x07ffffff, 0x03000000,
489 0x9060, 0x0000007f, 0x00000020,
490 0x9508, 0x00010000, 0x00010000,
491 0xac14, 0x000003ff, 0x000000f3,
492 0xac10, 0xffffffff, 0x00000000,
493 0xac0c, 0xffffffff, 0x00003210,
494 0x88d4, 0x0000001f, 0x00000010,
495 0x15c0, 0x000c0fc0, 0x000c0400
496};
497
fffbdda4
AD		 498static const u32 hainan_golden_registers[] =
499{
500 0x9a10, 0x00010000, 0x00018208,
501 0x9830, 0xffffffff, 0x00000000,
502 0x9834, 0xf00fffff, 0x00000400,
503 0x9838, 0x0002021c, 0x00020200,
504 0xd0c0, 0xff000fff, 0x00000100,
505 0xd030, 0x000300c0, 0x00800040,
506 0xd8c0, 0xff000fff, 0x00000100,
507 0xd830, 0x000300c0, 0x00800040,
508 0x2ae4, 0x00073ffe, 0x000022a2,
509 0x240c, 0x000007ff, 0x00000000,
510 0x8a14, 0xf000001f, 0x00000007,
511 0x8b24, 0xffffffff, 0x00ffffff,
512 0x8b10, 0x0000ff0f, 0x00000000,
513 0x28a4c, 0x07ffffff, 0x4e000000,
514 0x28350, 0x3f3f3fff, 0x00000000,
515 0x30, 0x000000ff, 0x0040,
516 0x34, 0x00000040, 0x00004040,
517 0x9100, 0x03e00000, 0x03600000,
518 0x9060, 0x0000007f, 0x00000020,
519 0x9508, 0x00010000, 0x00010000,
520 0xac14, 0x000003ff, 0x000000f1,
521 0xac10, 0xffffffff, 0x00000000,
522 0xac0c, 0xffffffff, 0x00003210,
523 0x88d4, 0x0000001f, 0x00000010,
524 0x15c0, 0x000c0fc0, 0x000c0400
525};
526
527static const u32 hainan_golden_registers2[] =
528{
529 0x98f8, 0xffffffff, 0x02010001
530};
531
205996c0
AD		 532static const u32 tahiti_mgcg_cgcg_init[] =
533{
534 0xc400, 0xffffffff, 0xfffffffc,
535 0x802c, 0xffffffff, 0xe0000000,
536 0x9a60, 0xffffffff, 0x00000100,
537 0x92a4, 0xffffffff, 0x00000100,
538 0xc164, 0xffffffff, 0x00000100,
539 0x9774, 0xffffffff, 0x00000100,
540 0x8984, 0xffffffff, 0x06000100,
541 0x8a18, 0xffffffff, 0x00000100,
542 0x92a0, 0xffffffff, 0x00000100,
543 0xc380, 0xffffffff, 0x00000100,
544 0x8b28, 0xffffffff, 0x00000100,
545 0x9144, 0xffffffff, 0x00000100,
546 0x8d88, 0xffffffff, 0x00000100,
547 0x8d8c, 0xffffffff, 0x00000100,
548 0x9030, 0xffffffff, 0x00000100,
549 0x9034, 0xffffffff, 0x00000100,
550 0x9038, 0xffffffff, 0x00000100,
551 0x903c, 0xffffffff, 0x00000100,
552 0xad80, 0xffffffff, 0x00000100,
553 0xac54, 0xffffffff, 0x00000100,
554 0x897c, 0xffffffff, 0x06000100,
555 0x9868, 0xffffffff, 0x00000100,
556 0x9510, 0xffffffff, 0x00000100,
557 0xaf04, 0xffffffff, 0x00000100,
558 0xae04, 0xffffffff, 0x00000100,
559 0x949c, 0xffffffff, 0x00000100,
560 0x802c, 0xffffffff, 0xe0000000,
561 0x9160, 0xffffffff, 0x00010000,
562 0x9164, 0xffffffff, 0x00030002,
563 0x9168, 0xffffffff, 0x00040007,
564 0x916c, 0xffffffff, 0x00060005,
565 0x9170, 0xffffffff, 0x00090008,
566 0x9174, 0xffffffff, 0x00020001,
567 0x9178, 0xffffffff, 0x00040003,
568 0x917c, 0xffffffff, 0x00000007,
569 0x9180, 0xffffffff, 0x00060005,
570 0x9184, 0xffffffff, 0x00090008,
571 0x9188, 0xffffffff, 0x00030002,
572 0x918c, 0xffffffff, 0x00050004,
573 0x9190, 0xffffffff, 0x00000008,
574 0x9194, 0xffffffff, 0x00070006,
575 0x9198, 0xffffffff, 0x000a0009,
576 0x919c, 0xffffffff, 0x00040003,
577 0x91a0, 0xffffffff, 0x00060005,
578 0x91a4, 0xffffffff, 0x00000009,
579 0x91a8, 0xffffffff, 0x00080007,
580 0x91ac, 0xffffffff, 0x000b000a,
581 0x91b0, 0xffffffff, 0x00050004,
582 0x91b4, 0xffffffff, 0x00070006,
583 0x91b8, 0xffffffff, 0x0008000b,
584 0x91bc, 0xffffffff, 0x000a0009,
585 0x91c0, 0xffffffff, 0x000d000c,
586 0x91c4, 0xffffffff, 0x00060005,
587 0x91c8, 0xffffffff, 0x00080007,
588 0x91cc, 0xffffffff, 0x0000000b,
589 0x91d0, 0xffffffff, 0x000a0009,
590 0x91d4, 0xffffffff, 0x000d000c,
591 0x91d8, 0xffffffff, 0x00070006,
592 0x91dc, 0xffffffff, 0x00090008,
593 0x91e0, 0xffffffff, 0x0000000c,
594 0x91e4, 0xffffffff, 0x000b000a,
595 0x91e8, 0xffffffff, 0x000e000d,
596 0x91ec, 0xffffffff, 0x00080007,
597 0x91f0, 0xffffffff, 0x000a0009,
598 0x91f4, 0xffffffff, 0x0000000d,
599 0x91f8, 0xffffffff, 0x000c000b,
600 0x91fc, 0xffffffff, 0x000f000e,
601 0x9200, 0xffffffff, 0x00090008,
602 0x9204, 0xffffffff, 0x000b000a,
603 0x9208, 0xffffffff, 0x000c000f,
604 0x920c, 0xffffffff, 0x000e000d,
605 0x9210, 0xffffffff, 0x00110010,
606 0x9214, 0xffffffff, 0x000a0009,
607 0x9218, 0xffffffff, 0x000c000b,
608 0x921c, 0xffffffff, 0x0000000f,
609 0x9220, 0xffffffff, 0x000e000d,
610 0x9224, 0xffffffff, 0x00110010,
611 0x9228, 0xffffffff, 0x000b000a,
612 0x922c, 0xffffffff, 0x000d000c,
613 0x9230, 0xffffffff, 0x00000010,
614 0x9234, 0xffffffff, 0x000f000e,
615 0x9238, 0xffffffff, 0x00120011,
616 0x923c, 0xffffffff, 0x000c000b,
617 0x9240, 0xffffffff, 0x000e000d,
618 0x9244, 0xffffffff, 0x00000011,
619 0x9248, 0xffffffff, 0x0010000f,
620 0x924c, 0xffffffff, 0x00130012,
621 0x9250, 0xffffffff, 0x000d000c,
622 0x9254, 0xffffffff, 0x000f000e,
623 0x9258, 0xffffffff, 0x00100013,
624 0x925c, 0xffffffff, 0x00120011,
625 0x9260, 0xffffffff, 0x00150014,
626 0x9264, 0xffffffff, 0x000e000d,
627 0x9268, 0xffffffff, 0x0010000f,
628 0x926c, 0xffffffff, 0x00000013,
629 0x9270, 0xffffffff, 0x00120011,
630 0x9274, 0xffffffff, 0x00150014,
631 0x9278, 0xffffffff, 0x000f000e,
632 0x927c, 0xffffffff, 0x00110010,
633 0x9280, 0xffffffff, 0x00000014,
634 0x9284, 0xffffffff, 0x00130012,
635 0x9288, 0xffffffff, 0x00160015,
636 0x928c, 0xffffffff, 0x0010000f,
637 0x9290, 0xffffffff, 0x00120011,
638 0x9294, 0xffffffff, 0x00000015,
639 0x9298, 0xffffffff, 0x00140013,
640 0x929c, 0xffffffff, 0x00170016,
641 0x9150, 0xffffffff, 0x96940200,
642 0x8708, 0xffffffff, 0x00900100,
643 0xc478, 0xffffffff, 0x00000080,
644 0xc404, 0xffffffff, 0x0020003f,
645 0x30, 0xffffffff, 0x0000001c,
646 0x34, 0x000f0000, 0x000f0000,
647 0x160c, 0xffffffff, 0x00000100,
648 0x1024, 0xffffffff, 0x00000100,
649 0x102c, 0x00000101, 0x00000000,
650 0x20a8, 0xffffffff, 0x00000104,
651 0x264c, 0x000c0000, 0x000c0000,
652 0x2648, 0x000c0000, 0x000c0000,
653 0x55e4, 0xff000fff, 0x00000100,
654 0x55e8, 0x00000001, 0x00000001,
655 0x2f50, 0x00000001, 0x00000001,
656 0x30cc, 0xc0000fff, 0x00000104,
657 0xc1e4, 0x00000001, 0x00000001,
658 0xd0c0, 0xfffffff0, 0x00000100,
659 0xd8c0, 0xfffffff0, 0x00000100
660};
661
662static const u32 pitcairn_mgcg_cgcg_init[] =
663{
664 0xc400, 0xffffffff, 0xfffffffc,
665 0x802c, 0xffffffff, 0xe0000000,
666 0x9a60, 0xffffffff, 0x00000100,
667 0x92a4, 0xffffffff, 0x00000100,
668 0xc164, 0xffffffff, 0x00000100,
669 0x9774, 0xffffffff, 0x00000100,
670 0x8984, 0xffffffff, 0x06000100,
671 0x8a18, 0xffffffff, 0x00000100,
672 0x92a0, 0xffffffff, 0x00000100,
673 0xc380, 0xffffffff, 0x00000100,
674 0x8b28, 0xffffffff, 0x00000100,
675 0x9144, 0xffffffff, 0x00000100,
676 0x8d88, 0xffffffff, 0x00000100,
677 0x8d8c, 0xffffffff, 0x00000100,
678 0x9030, 0xffffffff, 0x00000100,
679 0x9034, 0xffffffff, 0x00000100,
680 0x9038, 0xffffffff, 0x00000100,
681 0x903c, 0xffffffff, 0x00000100,
682 0xad80, 0xffffffff, 0x00000100,
683 0xac54, 0xffffffff, 0x00000100,
684 0x897c, 0xffffffff, 0x06000100,
685 0x9868, 0xffffffff, 0x00000100,
686 0x9510, 0xffffffff, 0x00000100,
687 0xaf04, 0xffffffff, 0x00000100,
688 0xae04, 0xffffffff, 0x00000100,
689 0x949c, 0xffffffff, 0x00000100,
690 0x802c, 0xffffffff, 0xe0000000,
691 0x9160, 0xffffffff, 0x00010000,
692 0x9164, 0xffffffff, 0x00030002,
693 0x9168, 0xffffffff, 0x00040007,
694 0x916c, 0xffffffff, 0x00060005,
695 0x9170, 0xffffffff, 0x00090008,
696 0x9174, 0xffffffff, 0x00020001,
697 0x9178, 0xffffffff, 0x00040003,
698 0x917c, 0xffffffff, 0x00000007,
699 0x9180, 0xffffffff, 0x00060005,
700 0x9184, 0xffffffff, 0x00090008,
701 0x9188, 0xffffffff, 0x00030002,
702 0x918c, 0xffffffff, 0x00050004,
703 0x9190, 0xffffffff, 0x00000008,
704 0x9194, 0xffffffff, 0x00070006,
705 0x9198, 0xffffffff, 0x000a0009,
706 0x919c, 0xffffffff, 0x00040003,
707 0x91a0, 0xffffffff, 0x00060005,
708 0x91a4, 0xffffffff, 0x00000009,
709 0x91a8, 0xffffffff, 0x00080007,
710 0x91ac, 0xffffffff, 0x000b000a,
711 0x91b0, 0xffffffff, 0x00050004,
712 0x91b4, 0xffffffff, 0x00070006,
713 0x91b8, 0xffffffff, 0x0008000b,
714 0x91bc, 0xffffffff, 0x000a0009,
715 0x91c0, 0xffffffff, 0x000d000c,
716 0x9200, 0xffffffff, 0x00090008,
717 0x9204, 0xffffffff, 0x000b000a,
718 0x9208, 0xffffffff, 0x000c000f,
719 0x920c, 0xffffffff, 0x000e000d,
720 0x9210, 0xffffffff, 0x00110010,
721 0x9214, 0xffffffff, 0x000a0009,
722 0x9218, 0xffffffff, 0x000c000b,
723 0x921c, 0xffffffff, 0x0000000f,
724 0x9220, 0xffffffff, 0x000e000d,
725 0x9224, 0xffffffff, 0x00110010,
726 0x9228, 0xffffffff, 0x000b000a,
727 0x922c, 0xffffffff, 0x000d000c,
728 0x9230, 0xffffffff, 0x00000010,
729 0x9234, 0xffffffff, 0x000f000e,
730 0x9238, 0xffffffff, 0x00120011,
731 0x923c, 0xffffffff, 0x000c000b,
732 0x9240, 0xffffffff, 0x000e000d,
733 0x9244, 0xffffffff, 0x00000011,
734 0x9248, 0xffffffff, 0x0010000f,
735 0x924c, 0xffffffff, 0x00130012,
736 0x9250, 0xffffffff, 0x000d000c,
737 0x9254, 0xffffffff, 0x000f000e,
738 0x9258, 0xffffffff, 0x00100013,
739 0x925c, 0xffffffff, 0x00120011,
740 0x9260, 0xffffffff, 0x00150014,
741 0x9150, 0xffffffff, 0x96940200,
742 0x8708, 0xffffffff, 0x00900100,
743 0xc478, 0xffffffff, 0x00000080,
744 0xc404, 0xffffffff, 0x0020003f,
745 0x30, 0xffffffff, 0x0000001c,
746 0x34, 0x000f0000, 0x000f0000,
747 0x160c, 0xffffffff, 0x00000100,
748 0x1024, 0xffffffff, 0x00000100,
749 0x102c, 0x00000101, 0x00000000,
750 0x20a8, 0xffffffff, 0x00000104,
751 0x55e4, 0xff000fff, 0x00000100,
752 0x55e8, 0x00000001, 0x00000001,
753 0x2f50, 0x00000001, 0x00000001,
754 0x30cc, 0xc0000fff, 0x00000104,
755 0xc1e4, 0x00000001, 0x00000001,
756 0xd0c0, 0xfffffff0, 0x00000100,
757 0xd8c0, 0xfffffff0, 0x00000100
758};
759
760static const u32 verde_mgcg_cgcg_init[] =
761{
762 0xc400, 0xffffffff, 0xfffffffc,
763 0x802c, 0xffffffff, 0xe0000000,
764 0x9a60, 0xffffffff, 0x00000100,
765 0x92a4, 0xffffffff, 0x00000100,
766 0xc164, 0xffffffff, 0x00000100,
767 0x9774, 0xffffffff, 0x00000100,
768 0x8984, 0xffffffff, 0x06000100,
769 0x8a18, 0xffffffff, 0x00000100,
770 0x92a0, 0xffffffff, 0x00000100,
771 0xc380, 0xffffffff, 0x00000100,
772 0x8b28, 0xffffffff, 0x00000100,
773 0x9144, 0xffffffff, 0x00000100,
774 0x8d88, 0xffffffff, 0x00000100,
775 0x8d8c, 0xffffffff, 0x00000100,
776 0x9030, 0xffffffff, 0x00000100,
777 0x9034, 0xffffffff, 0x00000100,
778 0x9038, 0xffffffff, 0x00000100,
779 0x903c, 0xffffffff, 0x00000100,
780 0xad80, 0xffffffff, 0x00000100,
781 0xac54, 0xffffffff, 0x00000100,
782 0x897c, 0xffffffff, 0x06000100,
783 0x9868, 0xffffffff, 0x00000100,
784 0x9510, 0xffffffff, 0x00000100,
785 0xaf04, 0xffffffff, 0x00000100,
786 0xae04, 0xffffffff, 0x00000100,
787 0x949c, 0xffffffff, 0x00000100,
788 0x802c, 0xffffffff, 0xe0000000,
789 0x9160, 0xffffffff, 0x00010000,
790 0x9164, 0xffffffff, 0x00030002,
791 0x9168, 0xffffffff, 0x00040007,
792 0x916c, 0xffffffff, 0x00060005,
793 0x9170, 0xffffffff, 0x00090008,
794 0x9174, 0xffffffff, 0x00020001,
795 0x9178, 0xffffffff, 0x00040003,
796 0x917c, 0xffffffff, 0x00000007,
797 0x9180, 0xffffffff, 0x00060005,
798 0x9184, 0xffffffff, 0x00090008,
799 0x9188, 0xffffffff, 0x00030002,
800 0x918c, 0xffffffff, 0x00050004,
801 0x9190, 0xffffffff, 0x00000008,
802 0x9194, 0xffffffff, 0x00070006,
803 0x9198, 0xffffffff, 0x000a0009,
804 0x919c, 0xffffffff, 0x00040003,
805 0x91a0, 0xffffffff, 0x00060005,
806 0x91a4, 0xffffffff, 0x00000009,
807 0x91a8, 0xffffffff, 0x00080007,
808 0x91ac, 0xffffffff, 0x000b000a,
809 0x91b0, 0xffffffff, 0x00050004,
810 0x91b4, 0xffffffff, 0x00070006,
811 0x91b8, 0xffffffff, 0x0008000b,
812 0x91bc, 0xffffffff, 0x000a0009,
813 0x91c0, 0xffffffff, 0x000d000c,
814 0x9200, 0xffffffff, 0x00090008,
815 0x9204, 0xffffffff, 0x000b000a,
816 0x9208, 0xffffffff, 0x000c000f,
817 0x920c, 0xffffffff, 0x000e000d,
818 0x9210, 0xffffffff, 0x00110010,
819 0x9214, 0xffffffff, 0x000a0009,
820 0x9218, 0xffffffff, 0x000c000b,
821 0x921c, 0xffffffff, 0x0000000f,
822 0x9220, 0xffffffff, 0x000e000d,
823 0x9224, 0xffffffff, 0x00110010,
824 0x9228, 0xffffffff, 0x000b000a,
825 0x922c, 0xffffffff, 0x000d000c,
826 0x9230, 0xffffffff, 0x00000010,
827 0x9234, 0xffffffff, 0x000f000e,
828 0x9238, 0xffffffff, 0x00120011,
829 0x923c, 0xffffffff, 0x000c000b,
830 0x9240, 0xffffffff, 0x000e000d,
831 0x9244, 0xffffffff, 0x00000011,
832 0x9248, 0xffffffff, 0x0010000f,
833 0x924c, 0xffffffff, 0x00130012,
834 0x9250, 0xffffffff, 0x000d000c,
835 0x9254, 0xffffffff, 0x000f000e,
836 0x9258, 0xffffffff, 0x00100013,
837 0x925c, 0xffffffff, 0x00120011,
838 0x9260, 0xffffffff, 0x00150014,
839 0x9150, 0xffffffff, 0x96940200,
840 0x8708, 0xffffffff, 0x00900100,
841 0xc478, 0xffffffff, 0x00000080,
842 0xc404, 0xffffffff, 0x0020003f,
843 0x30, 0xffffffff, 0x0000001c,
844 0x34, 0x000f0000, 0x000f0000,
845 0x160c, 0xffffffff, 0x00000100,
846 0x1024, 0xffffffff, 0x00000100,
847 0x102c, 0x00000101, 0x00000000,
848 0x20a8, 0xffffffff, 0x00000104,
849 0x264c, 0x000c0000, 0x000c0000,
850 0x2648, 0x000c0000, 0x000c0000,
851 0x55e4, 0xff000fff, 0x00000100,
852 0x55e8, 0x00000001, 0x00000001,
853 0x2f50, 0x00000001, 0x00000001,
854 0x30cc, 0xc0000fff, 0x00000104,
855 0xc1e4, 0x00000001, 0x00000001,
856 0xd0c0, 0xfffffff0, 0x00000100,
857 0xd8c0, 0xfffffff0, 0x00000100
858};
859
860static const u32 oland_mgcg_cgcg_init[] =
861{
862 0xc400, 0xffffffff, 0xfffffffc,
863 0x802c, 0xffffffff, 0xe0000000,
864 0x9a60, 0xffffffff, 0x00000100,
865 0x92a4, 0xffffffff, 0x00000100,
866 0xc164, 0xffffffff, 0x00000100,
867 0x9774, 0xffffffff, 0x00000100,
868 0x8984, 0xffffffff, 0x06000100,
869 0x8a18, 0xffffffff, 0x00000100,
870 0x92a0, 0xffffffff, 0x00000100,
871 0xc380, 0xffffffff, 0x00000100,
872 0x8b28, 0xffffffff, 0x00000100,
873 0x9144, 0xffffffff, 0x00000100,
874 0x8d88, 0xffffffff, 0x00000100,
875 0x8d8c, 0xffffffff, 0x00000100,
876 0x9030, 0xffffffff, 0x00000100,
877 0x9034, 0xffffffff, 0x00000100,
878 0x9038, 0xffffffff, 0x00000100,
879 0x903c, 0xffffffff, 0x00000100,
880 0xad80, 0xffffffff, 0x00000100,
881 0xac54, 0xffffffff, 0x00000100,
882 0x897c, 0xffffffff, 0x06000100,
883 0x9868, 0xffffffff, 0x00000100,
884 0x9510, 0xffffffff, 0x00000100,
885 0xaf04, 0xffffffff, 0x00000100,
886 0xae04, 0xffffffff, 0x00000100,
887 0x949c, 0xffffffff, 0x00000100,
888 0x802c, 0xffffffff, 0xe0000000,
889 0x9160, 0xffffffff, 0x00010000,
890 0x9164, 0xffffffff, 0x00030002,
891 0x9168, 0xffffffff, 0x00040007,
892 0x916c, 0xffffffff, 0x00060005,
893 0x9170, 0xffffffff, 0x00090008,
894 0x9174, 0xffffffff, 0x00020001,
895 0x9178, 0xffffffff, 0x00040003,
896 0x917c, 0xffffffff, 0x00000007,
897 0x9180, 0xffffffff, 0x00060005,
898 0x9184, 0xffffffff, 0x00090008,
899 0x9188, 0xffffffff, 0x00030002,
900 0x918c, 0xffffffff, 0x00050004,
901 0x9190, 0xffffffff, 0x00000008,
902 0x9194, 0xffffffff, 0x00070006,
903 0x9198, 0xffffffff, 0x000a0009,
904 0x919c, 0xffffffff, 0x00040003,
905 0x91a0, 0xffffffff, 0x00060005,
906 0x91a4, 0xffffffff, 0x00000009,
907 0x91a8, 0xffffffff, 0x00080007,
908 0x91ac, 0xffffffff, 0x000b000a,
909 0x91b0, 0xffffffff, 0x00050004,
910 0x91b4, 0xffffffff, 0x00070006,
911 0x91b8, 0xffffffff, 0x0008000b,
912 0x91bc, 0xffffffff, 0x000a0009,
913 0x91c0, 0xffffffff, 0x000d000c,
914 0x91c4, 0xffffffff, 0x00060005,
915 0x91c8, 0xffffffff, 0x00080007,
916 0x91cc, 0xffffffff, 0x0000000b,
917 0x91d0, 0xffffffff, 0x000a0009,
918 0x91d4, 0xffffffff, 0x000d000c,
919 0x9150, 0xffffffff, 0x96940200,
920 0x8708, 0xffffffff, 0x00900100,
921 0xc478, 0xffffffff, 0x00000080,
922 0xc404, 0xffffffff, 0x0020003f,
923 0x30, 0xffffffff, 0x0000001c,
924 0x34, 0x000f0000, 0x000f0000,
925 0x160c, 0xffffffff, 0x00000100,
926 0x1024, 0xffffffff, 0x00000100,
927 0x102c, 0x00000101, 0x00000000,
928 0x20a8, 0xffffffff, 0x00000104,
929 0x264c, 0x000c0000, 0x000c0000,
930 0x2648, 0x000c0000, 0x000c0000,
931 0x55e4, 0xff000fff, 0x00000100,
932 0x55e8, 0x00000001, 0x00000001,
933 0x2f50, 0x00000001, 0x00000001,
934 0x30cc, 0xc0000fff, 0x00000104,
935 0xc1e4, 0x00000001, 0x00000001,
936 0xd0c0, 0xfffffff0, 0x00000100,
937 0xd8c0, 0xfffffff0, 0x00000100
938};
939
fffbdda4
AD		 940static const u32 hainan_mgcg_cgcg_init[] =
941{
942 0xc400, 0xffffffff, 0xfffffffc,
943 0x802c, 0xffffffff, 0xe0000000,
944 0x9a60, 0xffffffff, 0x00000100,
945 0x92a4, 0xffffffff, 0x00000100,
946 0xc164, 0xffffffff, 0x00000100,
947 0x9774, 0xffffffff, 0x00000100,
948 0x8984, 0xffffffff, 0x06000100,
949 0x8a18, 0xffffffff, 0x00000100,
950 0x92a0, 0xffffffff, 0x00000100,
951 0xc380, 0xffffffff, 0x00000100,
952 0x8b28, 0xffffffff, 0x00000100,
953 0x9144, 0xffffffff, 0x00000100,
954 0x8d88, 0xffffffff, 0x00000100,
955 0x8d8c, 0xffffffff, 0x00000100,
956 0x9030, 0xffffffff, 0x00000100,
957 0x9034, 0xffffffff, 0x00000100,
958 0x9038, 0xffffffff, 0x00000100,
959 0x903c, 0xffffffff, 0x00000100,
960 0xad80, 0xffffffff, 0x00000100,
961 0xac54, 0xffffffff, 0x00000100,
962 0x897c, 0xffffffff, 0x06000100,
963 0x9868, 0xffffffff, 0x00000100,
964 0x9510, 0xffffffff, 0x00000100,
965 0xaf04, 0xffffffff, 0x00000100,
966 0xae04, 0xffffffff, 0x00000100,
967 0x949c, 0xffffffff, 0x00000100,
968 0x802c, 0xffffffff, 0xe0000000,
969 0x9160, 0xffffffff, 0x00010000,
970 0x9164, 0xffffffff, 0x00030002,
971 0x9168, 0xffffffff, 0x00040007,
972 0x916c, 0xffffffff, 0x00060005,
973 0x9170, 0xffffffff, 0x00090008,
974 0x9174, 0xffffffff, 0x00020001,
975 0x9178, 0xffffffff, 0x00040003,
976 0x917c, 0xffffffff, 0x00000007,
977 0x9180, 0xffffffff, 0x00060005,
978 0x9184, 0xffffffff, 0x00090008,
979 0x9188, 0xffffffff, 0x00030002,
980 0x918c, 0xffffffff, 0x00050004,
981 0x9190, 0xffffffff, 0x00000008,
982 0x9194, 0xffffffff, 0x00070006,
983 0x9198, 0xffffffff, 0x000a0009,
984 0x919c, 0xffffffff, 0x00040003,
985 0x91a0, 0xffffffff, 0x00060005,
986 0x91a4, 0xffffffff, 0x00000009,
987 0x91a8, 0xffffffff, 0x00080007,
988 0x91ac, 0xffffffff, 0x000b000a,
989 0x91b0, 0xffffffff, 0x00050004,
990 0x91b4, 0xffffffff, 0x00070006,
991 0x91b8, 0xffffffff, 0x0008000b,
992 0x91bc, 0xffffffff, 0x000a0009,
993 0x91c0, 0xffffffff, 0x000d000c,
994 0x91c4, 0xffffffff, 0x00060005,
995 0x91c8, 0xffffffff, 0x00080007,
996 0x91cc, 0xffffffff, 0x0000000b,
997 0x91d0, 0xffffffff, 0x000a0009,
998 0x91d4, 0xffffffff, 0x000d000c,
999 0x9150, 0xffffffff, 0x96940200,
1000 0x8708, 0xffffffff, 0x00900100,
1001 0xc478, 0xffffffff, 0x00000080,
1002 0xc404, 0xffffffff, 0x0020003f,
1003 0x30, 0xffffffff, 0x0000001c,
1004 0x34, 0x000f0000, 0x000f0000,
1005 0x160c, 0xffffffff, 0x00000100,
1006 0x1024, 0xffffffff, 0x00000100,
1007 0x20a8, 0xffffffff, 0x00000104,
1008 0x264c, 0x000c0000, 0x000c0000,
1009 0x2648, 0x000c0000, 0x000c0000,
1010 0x2f50, 0x00000001, 0x00000001,
1011 0x30cc, 0xc0000fff, 0x00000104,
1012 0xc1e4, 0x00000001, 0x00000001,
1013 0xd0c0, 0xfffffff0, 0x00000100,
1014 0xd8c0, 0xfffffff0, 0x00000100
1015};
1016
205996c0
AD		 1017static u32 verde_pg_init[] =
1018{
1019 0x353c, 0xffffffff, 0x40000,
1020 0x3538, 0xffffffff, 0x200010ff,
1021 0x353c, 0xffffffff, 0x0,
1022 0x353c, 0xffffffff, 0x0,
1023 0x353c, 0xffffffff, 0x0,
1024 0x353c, 0xffffffff, 0x0,
1025 0x353c, 0xffffffff, 0x0,
1026 0x353c, 0xffffffff, 0x7007,
1027 0x3538, 0xffffffff, 0x300010ff,
1028 0x353c, 0xffffffff, 0x0,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x400000,
1034 0x3538, 0xffffffff, 0x100010ff,
1035 0x353c, 0xffffffff, 0x0,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x120200,
1041 0x3538, 0xffffffff, 0x500010ff,
1042 0x353c, 0xffffffff, 0x0,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x1e1e16,
1048 0x3538, 0xffffffff, 0x600010ff,
1049 0x353c, 0xffffffff, 0x0,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x171f1e,
1055 0x3538, 0xffffffff, 0x700010ff,
1056 0x353c, 0xffffffff, 0x0,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x3538, 0xffffffff, 0x9ff,
1063 0x3500, 0xffffffff, 0x0,
1064 0x3504, 0xffffffff, 0x10000800,
1065 0x3504, 0xffffffff, 0xf,
1066 0x3504, 0xffffffff, 0xf,
1067 0x3500, 0xffffffff, 0x4,
1068 0x3504, 0xffffffff, 0x1000051e,
1069 0x3504, 0xffffffff, 0xffff,
1070 0x3504, 0xffffffff, 0xffff,
1071 0x3500, 0xffffffff, 0x8,
1072 0x3504, 0xffffffff, 0x80500,
1073 0x3500, 0xffffffff, 0x12,
1074 0x3504, 0xffffffff, 0x9050c,
1075 0x3500, 0xffffffff, 0x1d,
1076 0x3504, 0xffffffff, 0xb052c,
1077 0x3500, 0xffffffff, 0x2a,
1078 0x3504, 0xffffffff, 0x1053e,
1079 0x3500, 0xffffffff, 0x2d,
1080 0x3504, 0xffffffff, 0x10546,
1081 0x3500, 0xffffffff, 0x30,
1082 0x3504, 0xffffffff, 0xa054e,
1083 0x3500, 0xffffffff, 0x3c,
1084 0x3504, 0xffffffff, 0x1055f,
1085 0x3500, 0xffffffff, 0x3f,
1086 0x3504, 0xffffffff, 0x10567,
1087 0x3500, 0xffffffff, 0x42,
1088 0x3504, 0xffffffff, 0x1056f,
1089 0x3500, 0xffffffff, 0x45,
1090 0x3504, 0xffffffff, 0x10572,
1091 0x3500, 0xffffffff, 0x48,
1092 0x3504, 0xffffffff, 0x20575,
1093 0x3500, 0xffffffff, 0x4c,
1094 0x3504, 0xffffffff, 0x190801,
1095 0x3500, 0xffffffff, 0x67,
1096 0x3504, 0xffffffff, 0x1082a,
1097 0x3500, 0xffffffff, 0x6a,
1098 0x3504, 0xffffffff, 0x1b082d,
1099 0x3500, 0xffffffff, 0x87,
1100 0x3504, 0xffffffff, 0x310851,
1101 0x3500, 0xffffffff, 0xba,
1102 0x3504, 0xffffffff, 0x891,
1103 0x3500, 0xffffffff, 0xbc,
1104 0x3504, 0xffffffff, 0x893,
1105 0x3500, 0xffffffff, 0xbe,
1106 0x3504, 0xffffffff, 0x20895,
1107 0x3500, 0xffffffff, 0xc2,
1108 0x3504, 0xffffffff, 0x20899,
1109 0x3500, 0xffffffff, 0xc6,
1110 0x3504, 0xffffffff, 0x2089d,
1111 0x3500, 0xffffffff, 0xca,
1112 0x3504, 0xffffffff, 0x8a1,
1113 0x3500, 0xffffffff, 0xcc,
1114 0x3504, 0xffffffff, 0x8a3,
1115 0x3500, 0xffffffff, 0xce,
1116 0x3504, 0xffffffff, 0x308a5,
1117 0x3500, 0xffffffff, 0xd3,
1118 0x3504, 0xffffffff, 0x6d08cd,
1119 0x3500, 0xffffffff, 0x142,
1120 0x3504, 0xffffffff, 0x2000095a,
1121 0x3504, 0xffffffff, 0x1,
1122 0x3500, 0xffffffff, 0x144,
1123 0x3504, 0xffffffff, 0x301f095b,
1124 0x3500, 0xffffffff, 0x165,
1125 0x3504, 0xffffffff, 0xc094d,
1126 0x3500, 0xffffffff, 0x173,
1127 0x3504, 0xffffffff, 0xf096d,
1128 0x3500, 0xffffffff, 0x184,
1129 0x3504, 0xffffffff, 0x15097f,
1130 0x3500, 0xffffffff, 0x19b,
1131 0x3504, 0xffffffff, 0xc0998,
1132 0x3500, 0xffffffff, 0x1a9,
1133 0x3504, 0xffffffff, 0x409a7,
1134 0x3500, 0xffffffff, 0x1af,
1135 0x3504, 0xffffffff, 0xcdc,
1136 0x3500, 0xffffffff, 0x1b1,
1137 0x3504, 0xffffffff, 0x800,
1138 0x3508, 0xffffffff, 0x6c9b2000,
1139 0x3510, 0xfc00, 0x2000,
1140 0x3544, 0xffffffff, 0xfc0,
1141 0x28d4, 0x00000100, 0x100
1142};
1143
1144static void si_init_golden_registers(struct radeon_device *rdev)
1145{
1146 switch (rdev->family) {
1147 case CHIP_TAHITI:
1148 radeon_program_register_sequence(rdev,
1149 tahiti_golden_registers,
1150 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1151 radeon_program_register_sequence(rdev,
1152 tahiti_golden_rlc_registers,
1153 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1154 radeon_program_register_sequence(rdev,
1155 tahiti_mgcg_cgcg_init,
1156 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1157 radeon_program_register_sequence(rdev,
1158 tahiti_golden_registers2,
1159 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1160 break;
1161 case CHIP_PITCAIRN:
1162 radeon_program_register_sequence(rdev,
1163 pitcairn_golden_registers,
1164 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1165 radeon_program_register_sequence(rdev,
1166 pitcairn_golden_rlc_registers,
1167 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1168 radeon_program_register_sequence(rdev,
1169 pitcairn_mgcg_cgcg_init,
1170 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1171 break;
1172 case CHIP_VERDE:
1173 radeon_program_register_sequence(rdev,
1174 verde_golden_registers,
1175 (const u32)ARRAY_SIZE(verde_golden_registers));
1176 radeon_program_register_sequence(rdev,
1177 verde_golden_rlc_registers,
1178 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1179 radeon_program_register_sequence(rdev,
1180 verde_mgcg_cgcg_init,
1181 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1182 radeon_program_register_sequence(rdev,
1183 verde_pg_init,
1184 (const u32)ARRAY_SIZE(verde_pg_init));
1185 break;
1186 case CHIP_OLAND:
1187 radeon_program_register_sequence(rdev,
1188 oland_golden_registers,
1189 (const u32)ARRAY_SIZE(oland_golden_registers));
1190 radeon_program_register_sequence(rdev,
1191 oland_golden_rlc_registers,
1192 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1193 radeon_program_register_sequence(rdev,
1194 oland_mgcg_cgcg_init,
1195 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1196 break;
fffbdda4
AD		 1197 case CHIP_HAINAN:
1198 radeon_program_register_sequence(rdev,
1199 hainan_golden_registers,
1200 (const u32)ARRAY_SIZE(hainan_golden_registers));
1201 radeon_program_register_sequence(rdev,
1202 hainan_golden_registers2,
1203 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1204 radeon_program_register_sequence(rdev,
1205 hainan_mgcg_cgcg_init,
1206 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1207 break;
205996c0
AD		 1208 default:
1209 break;
1210 }
1211}
1212
454d2e2a
AD		 1213#define PCIE_BUS_CLK 10000
1214#define TCLK (PCIE_BUS_CLK / 10)
1215
1216/**
1217 * si_get_xclk - get the xclk
1218 *
1219 * @rdev: radeon_device pointer
1220 *
1221 * Returns the reference clock used by the gfx engine
1222 * (SI).
1223 */
1224u32 si_get_xclk(struct radeon_device *rdev)
1225{
1226 u32 reference_clock = rdev->clock.spll.reference_freq;
1227 u32 tmp;
1228
1229 tmp = RREG32(CG_CLKPIN_CNTL_2);
1230 if (tmp & MUX_TCLK_TO_XCLK)
1231 return TCLK;
1232
1233 tmp = RREG32(CG_CLKPIN_CNTL);
1234 if (tmp & XTALIN_DIVIDE)
1235 return reference_clock / 4;
1236
1237 return reference_clock;
1238}
1239
1bd47d2e
AD		 1240/* get temperature in millidegrees */
1241int si_get_temp(struct radeon_device *rdev)
1242{
1243 u32 temp;
1244 int actual_temp = 0;
1245
1246 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1247 CTF_TEMP_SHIFT;
1248
1249 if (temp & 0x200)
1250 actual_temp = 255;
1251 else
1252 actual_temp = temp & 0x1ff;
1253
1254 actual_temp = (actual_temp * 1000);
1255
1256 return actual_temp;
1257}
1258
8b074dd6
AD		 1259#define TAHITI_IO_MC_REGS_SIZE 36
1260
1261static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1262 {0x0000006f, 0x03044000},
1263 {0x00000070, 0x0480c018},
1264 {0x00000071, 0x00000040},
1265 {0x00000072, 0x01000000},
1266 {0x00000074, 0x000000ff},
1267 {0x00000075, 0x00143400},
1268 {0x00000076, 0x08ec0800},
1269 {0x00000077, 0x040000cc},
1270 {0x00000079, 0x00000000},
1271 {0x0000007a, 0x21000409},
1272 {0x0000007c, 0x00000000},
1273 {0x0000007d, 0xe8000000},
1274 {0x0000007e, 0x044408a8},
1275 {0x0000007f, 0x00000003},
1276 {0x00000080, 0x00000000},
1277 {0x00000081, 0x01000000},
1278 {0x00000082, 0x02000000},
1279 {0x00000083, 0x00000000},
1280 {0x00000084, 0xe3f3e4f4},
1281 {0x00000085, 0x00052024},
1282 {0x00000087, 0x00000000},
1283 {0x00000088, 0x66036603},
1284 {0x00000089, 0x01000000},
1285 {0x0000008b, 0x1c0a0000},
1286 {0x0000008c, 0xff010000},
1287 {0x0000008e, 0xffffefff},
1288 {0x0000008f, 0xfff3efff},
1289 {0x00000090, 0xfff3efbf},
1290 {0x00000094, 0x00101101},
1291 {0x00000095, 0x00000fff},
1292 {0x00000096, 0x00116fff},
1293 {0x00000097, 0x60010000},
1294 {0x00000098, 0x10010000},
1295 {0x00000099, 0x00006000},
1296 {0x0000009a, 0x00001000},
1297 {0x0000009f, 0x00a77400}
1298};
1299
1300static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1301 {0x0000006f, 0x03044000},
1302 {0x00000070, 0x0480c018},
1303 {0x00000071, 0x00000040},
1304 {0x00000072, 0x01000000},
1305 {0x00000074, 0x000000ff},
1306 {0x00000075, 0x00143400},
1307 {0x00000076, 0x08ec0800},
1308 {0x00000077, 0x040000cc},
1309 {0x00000079, 0x00000000},
1310 {0x0000007a, 0x21000409},
1311 {0x0000007c, 0x00000000},
1312 {0x0000007d, 0xe8000000},
1313 {0x0000007e, 0x044408a8},
1314 {0x0000007f, 0x00000003},
1315 {0x00000080, 0x00000000},
1316 {0x00000081, 0x01000000},
1317 {0x00000082, 0x02000000},
1318 {0x00000083, 0x00000000},
1319 {0x00000084, 0xe3f3e4f4},
1320 {0x00000085, 0x00052024},
1321 {0x00000087, 0x00000000},
1322 {0x00000088, 0x66036603},
1323 {0x00000089, 0x01000000},
1324 {0x0000008b, 0x1c0a0000},
1325 {0x0000008c, 0xff010000},
1326 {0x0000008e, 0xffffefff},
1327 {0x0000008f, 0xfff3efff},
1328 {0x00000090, 0xfff3efbf},
1329 {0x00000094, 0x00101101},
1330 {0x00000095, 0x00000fff},
1331 {0x00000096, 0x00116fff},
1332 {0x00000097, 0x60010000},
1333 {0x00000098, 0x10010000},
1334 {0x00000099, 0x00006000},
1335 {0x0000009a, 0x00001000},
1336 {0x0000009f, 0x00a47400}
1337};
1338
1339static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1340 {0x0000006f, 0x03044000},
1341 {0x00000070, 0x0480c018},
1342 {0x00000071, 0x00000040},
1343 {0x00000072, 0x01000000},
1344 {0x00000074, 0x000000ff},
1345 {0x00000075, 0x00143400},
1346 {0x00000076, 0x08ec0800},
1347 {0x00000077, 0x040000cc},
1348 {0x00000079, 0x00000000},
1349 {0x0000007a, 0x21000409},
1350 {0x0000007c, 0x00000000},
1351 {0x0000007d, 0xe8000000},
1352 {0x0000007e, 0x044408a8},
1353 {0x0000007f, 0x00000003},
1354 {0x00000080, 0x00000000},
1355 {0x00000081, 0x01000000},
1356 {0x00000082, 0x02000000},
1357 {0x00000083, 0x00000000},
1358 {0x00000084, 0xe3f3e4f4},
1359 {0x00000085, 0x00052024},
1360 {0x00000087, 0x00000000},
1361 {0x00000088, 0x66036603},
1362 {0x00000089, 0x01000000},
1363 {0x0000008b, 0x1c0a0000},
1364 {0x0000008c, 0xff010000},
1365 {0x0000008e, 0xffffefff},
1366 {0x0000008f, 0xfff3efff},
1367 {0x00000090, 0xfff3efbf},
1368 {0x00000094, 0x00101101},
1369 {0x00000095, 0x00000fff},
1370 {0x00000096, 0x00116fff},
1371 {0x00000097, 0x60010000},
1372 {0x00000098, 0x10010000},
1373 {0x00000099, 0x00006000},
1374 {0x0000009a, 0x00001000},
1375 {0x0000009f, 0x00a37400}
1376};
1377
bcc7f5d2
AD		 1378static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1379 {0x0000006f, 0x03044000},
1380 {0x00000070, 0x0480c018},
1381 {0x00000071, 0x00000040},
1382 {0x00000072, 0x01000000},
1383 {0x00000074, 0x000000ff},
1384 {0x00000075, 0x00143400},
1385 {0x00000076, 0x08ec0800},
1386 {0x00000077, 0x040000cc},
1387 {0x00000079, 0x00000000},
1388 {0x0000007a, 0x21000409},
1389 {0x0000007c, 0x00000000},
1390 {0x0000007d, 0xe8000000},
1391 {0x0000007e, 0x044408a8},
1392 {0x0000007f, 0x00000003},
1393 {0x00000080, 0x00000000},
1394 {0x00000081, 0x01000000},
1395 {0x00000082, 0x02000000},
1396 {0x00000083, 0x00000000},
1397 {0x00000084, 0xe3f3e4f4},
1398 {0x00000085, 0x00052024},
1399 {0x00000087, 0x00000000},
1400 {0x00000088, 0x66036603},
1401 {0x00000089, 0x01000000},
1402 {0x0000008b, 0x1c0a0000},
1403 {0x0000008c, 0xff010000},
1404 {0x0000008e, 0xffffefff},
1405 {0x0000008f, 0xfff3efff},
1406 {0x00000090, 0xfff3efbf},
1407 {0x00000094, 0x00101101},
1408 {0x00000095, 0x00000fff},
1409 {0x00000096, 0x00116fff},
1410 {0x00000097, 0x60010000},
1411 {0x00000098, 0x10010000},
1412 {0x00000099, 0x00006000},
1413 {0x0000009a, 0x00001000},
1414 {0x0000009f, 0x00a17730}
1415};
1416
c04c00b4
AD		 1417static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1418 {0x0000006f, 0x03044000},
1419 {0x00000070, 0x0480c018},
1420 {0x00000071, 0x00000040},
1421 {0x00000072, 0x01000000},
1422 {0x00000074, 0x000000ff},
1423 {0x00000075, 0x00143400},
1424 {0x00000076, 0x08ec0800},
1425 {0x00000077, 0x040000cc},
1426 {0x00000079, 0x00000000},
1427 {0x0000007a, 0x21000409},
1428 {0x0000007c, 0x00000000},
1429 {0x0000007d, 0xe8000000},
1430 {0x0000007e, 0x044408a8},
1431 {0x0000007f, 0x00000003},
1432 {0x00000080, 0x00000000},
1433 {0x00000081, 0x01000000},
1434 {0x00000082, 0x02000000},
1435 {0x00000083, 0x00000000},
1436 {0x00000084, 0xe3f3e4f4},
1437 {0x00000085, 0x00052024},
1438 {0x00000087, 0x00000000},
1439 {0x00000088, 0x66036603},
1440 {0x00000089, 0x01000000},
1441 {0x0000008b, 0x1c0a0000},
1442 {0x0000008c, 0xff010000},
1443 {0x0000008e, 0xffffefff},
1444 {0x0000008f, 0xfff3efff},
1445 {0x00000090, 0xfff3efbf},
1446 {0x00000094, 0x00101101},
1447 {0x00000095, 0x00000fff},
1448 {0x00000096, 0x00116fff},
1449 {0x00000097, 0x60010000},
1450 {0x00000098, 0x10010000},
1451 {0x00000099, 0x00006000},
1452 {0x0000009a, 0x00001000},
1453 {0x0000009f, 0x00a07730}
1454};
1455
8b074dd6
AD		 1456/* ucode loading */
1457static int si_mc_load_microcode(struct radeon_device *rdev)
1458{
1459 const __be32 *fw_data;
1460 u32 running, blackout = 0;
1461 u32 *io_mc_regs;
1462 int i, ucode_size, regs_size;
1463
1464 if (!rdev->mc_fw)
1465 return -EINVAL;
1466
1467 switch (rdev->family) {
1468 case CHIP_TAHITI:
1469 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1470 ucode_size = SI_MC_UCODE_SIZE;
1471 regs_size = TAHITI_IO_MC_REGS_SIZE;
1472 break;
1473 case CHIP_PITCAIRN:
1474 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1475 ucode_size = SI_MC_UCODE_SIZE;
1476 regs_size = TAHITI_IO_MC_REGS_SIZE;
1477 break;
1478 case CHIP_VERDE:
1479 default:
1480 io_mc_regs = (u32 *)&verde_io_mc_regs;
1481 ucode_size = SI_MC_UCODE_SIZE;
1482 regs_size = TAHITI_IO_MC_REGS_SIZE;
1483 break;
bcc7f5d2
AD		 1484 case CHIP_OLAND:
1485 io_mc_regs = (u32 *)&oland_io_mc_regs;
1486 ucode_size = OLAND_MC_UCODE_SIZE;
1487 regs_size = TAHITI_IO_MC_REGS_SIZE;
1488 break;
c04c00b4
AD		 1489 case CHIP_HAINAN:
1490 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1491 ucode_size = OLAND_MC_UCODE_SIZE;
1492 regs_size = TAHITI_IO_MC_REGS_SIZE;
1493 break;
8b074dd6
AD		 1494 }
1495
1496 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1497
1498 if (running == 0) {
1499 if (running) {
1500 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1501 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1502 }
1503
1504 /* reset the engine and set to writable */
1505 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1506 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1507
1508 /* load mc io regs */
1509 for (i = 0; i < regs_size; i++) {
1510 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1511 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1512 }
1513 /* load the MC ucode */
1514 fw_data = (const __be32 *)rdev->mc_fw->data;
1515 for (i = 0; i < ucode_size; i++)
1516 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1517
1518 /* put the engine back into the active state */
1519 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1520 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1521 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1522
1523 /* wait for training to complete */
1524 for (i = 0; i < rdev->usec_timeout; i++) {
1525 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1526 break;
1527 udelay(1);
1528 }
1529 for (i = 0; i < rdev->usec_timeout; i++) {
1530 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1531 break;
1532 udelay(1);
1533 }
1534
1535 if (running)
1536 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1537 }
1538
1539 return 0;
1540}
1541
0f0de06c
AD		 1542static int si_init_microcode(struct radeon_device *rdev)
1543{
1544 struct platform_device *pdev;
1545 const char *chip_name;
1546 const char *rlc_chip_name;
1547 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1548 char fw_name[30];
1549 int err;
1550
1551 DRM_DEBUG("\n");
1552
1553 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1554 err = IS_ERR(pdev);
1555 if (err) {
1556 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1557 return -EINVAL;
1558 }
1559
1560 switch (rdev->family) {
1561 case CHIP_TAHITI:
1562 chip_name = "TAHITI";
1563 rlc_chip_name = "TAHITI";
1564 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1565 me_req_size = SI_PM4_UCODE_SIZE * 4;
1566 ce_req_size = SI_CE_UCODE_SIZE * 4;
1567 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1568 mc_req_size = SI_MC_UCODE_SIZE * 4;
1569 break;
1570 case CHIP_PITCAIRN:
1571 chip_name = "PITCAIRN";
1572 rlc_chip_name = "PITCAIRN";
1573 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1574 me_req_size = SI_PM4_UCODE_SIZE * 4;
1575 ce_req_size = SI_CE_UCODE_SIZE * 4;
1576 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1577 mc_req_size = SI_MC_UCODE_SIZE * 4;
1578 break;
1579 case CHIP_VERDE:
1580 chip_name = "VERDE";
1581 rlc_chip_name = "VERDE";
1582 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1583 me_req_size = SI_PM4_UCODE_SIZE * 4;
1584 ce_req_size = SI_CE_UCODE_SIZE * 4;
1585 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1586 mc_req_size = SI_MC_UCODE_SIZE * 4;
1587 break;
bcc7f5d2
AD		 1588 case CHIP_OLAND:
1589 chip_name = "OLAND";
1590 rlc_chip_name = "OLAND";
1591 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1592 me_req_size = SI_PM4_UCODE_SIZE * 4;
1593 ce_req_size = SI_CE_UCODE_SIZE * 4;
1594 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1595 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1596 break;
c04c00b4
AD		 1597 case CHIP_HAINAN:
1598 chip_name = "HAINAN";
1599 rlc_chip_name = "HAINAN";
1600 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1601 me_req_size = SI_PM4_UCODE_SIZE * 4;
1602 ce_req_size = SI_CE_UCODE_SIZE * 4;
1603 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1604 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1605 break;
0f0de06c
AD		 1606 default: BUG();
1607 }
1608
1609 DRM_INFO("Loading %s Microcode\n", chip_name);
1610
1611 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1612 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1613 if (err)
1614 goto out;
1615 if (rdev->pfp_fw->size != pfp_req_size) {
1616 printk(KERN_ERR
1617 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1618 rdev->pfp_fw->size, fw_name);
1619 err = -EINVAL;
1620 goto out;
1621 }
1622
1623 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1624 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1625 if (err)
1626 goto out;
1627 if (rdev->me_fw->size != me_req_size) {
1628 printk(KERN_ERR
1629 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1630 rdev->me_fw->size, fw_name);
1631 err = -EINVAL;
1632 }
1633
1634 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1635 err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
1636 if (err)
1637 goto out;
1638 if (rdev->ce_fw->size != ce_req_size) {
1639 printk(KERN_ERR
1640 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1641 rdev->ce_fw->size, fw_name);
1642 err = -EINVAL;
1643 }
1644
1645 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1646 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1647 if (err)
1648 goto out;
1649 if (rdev->rlc_fw->size != rlc_req_size) {
1650 printk(KERN_ERR
1651 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1652 rdev->rlc_fw->size, fw_name);
1653 err = -EINVAL;
1654 }
1655
1656 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1657 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
1658 if (err)
1659 goto out;
1660 if (rdev->mc_fw->size != mc_req_size) {
1661 printk(KERN_ERR
1662 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1663 rdev->mc_fw->size, fw_name);
1664 err = -EINVAL;
1665 }
1666
1667out:
1668 platform_device_unregister(pdev);
1669
1670 if (err) {
1671 if (err != -EINVAL)
1672 printk(KERN_ERR
1673 "si_cp: Failed to load firmware \"%s\"\n",
1674 fw_name);
1675 release_firmware(rdev->pfp_fw);
1676 rdev->pfp_fw = NULL;
1677 release_firmware(rdev->me_fw);
1678 rdev->me_fw = NULL;
1679 release_firmware(rdev->ce_fw);
1680 rdev->ce_fw = NULL;
1681 release_firmware(rdev->rlc_fw);
1682 rdev->rlc_fw = NULL;
1683 release_firmware(rdev->mc_fw);
1684 rdev->mc_fw = NULL;
1685 }
1686 return err;
1687}
1688
43b3cd99
AD		 1689/* watermark setup */
1690static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1691 struct radeon_crtc *radeon_crtc,
1692 struct drm_display_mode *mode,
1693 struct drm_display_mode *other_mode)
1694{
1695 u32 tmp;
1696 /*
1697 * Line Buffer Setup
1698 * There are 3 line buffers, each one shared by 2 display controllers.
1699 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1700 * the display controllers. The paritioning is done via one of four
1701 * preset allocations specified in bits 21:20:
1702 * 0 - half lb
1703 * 2 - whole lb, other crtc must be disabled
1704 */
1705 /* this can get tricky if we have two large displays on a paired group
1706 * of crtcs. Ideally for multiple large displays we'd assign them to
1707 * non-linked crtcs for maximum line buffer allocation.
1708 */
1709 if (radeon_crtc->base.enabled && mode) {
1710 if (other_mode)
1711 tmp = 0; /* 1/2 */
1712 else
1713 tmp = 2; /* whole */
1714 } else
1715 tmp = 0;
1716
1717 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1718 DC_LB_MEMORY_CONFIG(tmp));
1719
1720 if (radeon_crtc->base.enabled && mode) {
1721 switch (tmp) {
1722 case 0:
1723 default:
1724 return 4096 * 2;
1725 case 2:
1726 return 8192 * 2;
1727 }
1728 }
1729
1730 /* controller not enabled, so no lb used */
1731 return 0;
1732}
1733
ca7db22b 1734static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1735{
1736 u32 tmp = RREG32(MC_SHARED_CHMAP);
1737
1738 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1739 case 0:
1740 default:
1741 return 1;
1742 case 1:
1743 return 2;
1744 case 2:
1745 return 4;
1746 case 3:
1747 return 8;
1748 case 4:
1749 return 3;
1750 case 5:
1751 return 6;
1752 case 6:
1753 return 10;
1754 case 7:
1755 return 12;
1756 case 8:
1757 return 16;
1758 }
1759}
1760
1761struct dce6_wm_params {
1762 u32 dram_channels; /* number of dram channels */
1763 u32 yclk; /* bandwidth per dram data pin in kHz */
1764 u32 sclk; /* engine clock in kHz */
1765 u32 disp_clk; /* display clock in kHz */
1766 u32 src_width; /* viewport width */
1767 u32 active_time; /* active display time in ns */
1768 u32 blank_time; /* blank time in ns */
1769 bool interlaced; /* mode is interlaced */
1770 fixed20_12 vsc; /* vertical scale ratio */
1771 u32 num_heads; /* number of active crtcs */
1772 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1773 u32 lb_size; /* line buffer allocated to pipe */
1774 u32 vtaps; /* vertical scaler taps */
1775};
1776
1777static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1778{
1779 /* Calculate raw DRAM Bandwidth */
1780 fixed20_12 dram_efficiency; /* 0.7 */
1781 fixed20_12 yclk, dram_channels, bandwidth;
1782 fixed20_12 a;
1783
1784 a.full = dfixed_const(1000);
1785 yclk.full = dfixed_const(wm->yclk);
1786 yclk.full = dfixed_div(yclk, a);
1787 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1788 a.full = dfixed_const(10);
1789 dram_efficiency.full = dfixed_const(7);
1790 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1791 bandwidth.full = dfixed_mul(dram_channels, yclk);
1792 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1793
1794 return dfixed_trunc(bandwidth);
1795}
1796
1797static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1798{
1799 /* Calculate DRAM Bandwidth and the part allocated to display. */
1800 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1801 fixed20_12 yclk, dram_channels, bandwidth;
1802 fixed20_12 a;
1803
1804 a.full = dfixed_const(1000);
1805 yclk.full = dfixed_const(wm->yclk);
1806 yclk.full = dfixed_div(yclk, a);
1807 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1808 a.full = dfixed_const(10);
1809 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1810 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1811 bandwidth.full = dfixed_mul(dram_channels, yclk);
1812 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1813
1814 return dfixed_trunc(bandwidth);
1815}
1816
1817static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1818{
1819 /* Calculate the display Data return Bandwidth */
1820 fixed20_12 return_efficiency; /* 0.8 */
1821 fixed20_12 sclk, bandwidth;
1822 fixed20_12 a;
1823
1824 a.full = dfixed_const(1000);
1825 sclk.full = dfixed_const(wm->sclk);
1826 sclk.full = dfixed_div(sclk, a);
1827 a.full = dfixed_const(10);
1828 return_efficiency.full = dfixed_const(8);
1829 return_efficiency.full = dfixed_div(return_efficiency, a);
1830 a.full = dfixed_const(32);
1831 bandwidth.full = dfixed_mul(a, sclk);
1832 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1833
1834 return dfixed_trunc(bandwidth);
1835}
1836
1837static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1838{
1839 return 32;
1840}
1841
1842static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1843{
1844 /* Calculate the DMIF Request Bandwidth */
1845 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1846 fixed20_12 disp_clk, sclk, bandwidth;
1847 fixed20_12 a, b1, b2;
1848 u32 min_bandwidth;
1849
1850 a.full = dfixed_const(1000);
1851 disp_clk.full = dfixed_const(wm->disp_clk);
1852 disp_clk.full = dfixed_div(disp_clk, a);
1853 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1854 b1.full = dfixed_mul(a, disp_clk);
1855
1856 a.full = dfixed_const(1000);
1857 sclk.full = dfixed_const(wm->sclk);
1858 sclk.full = dfixed_div(sclk, a);
1859 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1860 b2.full = dfixed_mul(a, sclk);
1861
1862 a.full = dfixed_const(10);
1863 disp_clk_request_efficiency.full = dfixed_const(8);
1864 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1865
1866 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1867
1868 a.full = dfixed_const(min_bandwidth);
1869 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1870
1871 return dfixed_trunc(bandwidth);
1872}
1873
1874static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1875{
1876 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1877 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1878 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1879 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1880
1881 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1882}
1883
1884static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1885{
1886 /* Calculate the display mode Average Bandwidth
1887 * DisplayMode should contain the source and destination dimensions,
1888 * timing, etc.
1889 */
1890 fixed20_12 bpp;
1891 fixed20_12 line_time;
1892 fixed20_12 src_width;
1893 fixed20_12 bandwidth;
1894 fixed20_12 a;
1895
1896 a.full = dfixed_const(1000);
1897 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1898 line_time.full = dfixed_div(line_time, a);
1899 bpp.full = dfixed_const(wm->bytes_per_pixel);
1900 src_width.full = dfixed_const(wm->src_width);
1901 bandwidth.full = dfixed_mul(src_width, bpp);
1902 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1903 bandwidth.full = dfixed_div(bandwidth, line_time);
1904
1905 return dfixed_trunc(bandwidth);
1906}
1907
1908static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1909{
1910 /* First calcualte the latency in ns */
1911 u32 mc_latency = 2000; /* 2000 ns. */
1912 u32 available_bandwidth = dce6_available_bandwidth(wm);
1913 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1914 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1915 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1916 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1917 (wm->num_heads * cursor_line_pair_return_time);
1918 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1919 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1920 u32 tmp, dmif_size = 12288;
1921 fixed20_12 a, b, c;
1922
1923 if (wm->num_heads == 0)
1924 return 0;
1925
1926 a.full = dfixed_const(2);
1927 b.full = dfixed_const(1);
1928 if ((wm->vsc.full > a.full) ||
1929 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1930 (wm->vtaps >= 5) ||
1931 ((wm->vsc.full >= a.full) && wm->interlaced))
1932 max_src_lines_per_dst_line = 4;
1933 else
1934 max_src_lines_per_dst_line = 2;
1935
1936 a.full = dfixed_const(available_bandwidth);
1937 b.full = dfixed_const(wm->num_heads);
1938 a.full = dfixed_div(a, b);
1939
1940 b.full = dfixed_const(mc_latency + 512);
1941 c.full = dfixed_const(wm->disp_clk);
1942 b.full = dfixed_div(b, c);
1943
1944 c.full = dfixed_const(dmif_size);
1945 b.full = dfixed_div(c, b);
1946
1947 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1948
1949 b.full = dfixed_const(1000);
1950 c.full = dfixed_const(wm->disp_clk);
1951 b.full = dfixed_div(c, b);
1952 c.full = dfixed_const(wm->bytes_per_pixel);
1953 b.full = dfixed_mul(b, c);
1954
1955 lb_fill_bw = min(tmp, dfixed_trunc(b));
1956
1957 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1958 b.full = dfixed_const(1000);
1959 c.full = dfixed_const(lb_fill_bw);
1960 b.full = dfixed_div(c, b);
1961 a.full = dfixed_div(a, b);
1962 line_fill_time = dfixed_trunc(a);
1963
1964 if (line_fill_time < wm->active_time)
1965 return latency;
1966 else
1967 return latency + (line_fill_time - wm->active_time);
1968
1969}
1970
1971static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1972{
1973 if (dce6_average_bandwidth(wm) <=
1974 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1975 return true;
1976 else
1977 return false;
1978};
1979
1980static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
1981{
1982 if (dce6_average_bandwidth(wm) <=
1983 (dce6_available_bandwidth(wm) / wm->num_heads))
1984 return true;
1985 else
1986 return false;
1987};
1988
1989static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
1990{
1991 u32 lb_partitions = wm->lb_size / wm->src_width;
1992 u32 line_time = wm->active_time + wm->blank_time;
1993 u32 latency_tolerant_lines;
1994 u32 latency_hiding;
1995 fixed20_12 a;
1996
1997 a.full = dfixed_const(1);
1998 if (wm->vsc.full > a.full)
1999 latency_tolerant_lines = 1;
2000 else {
2001 if (lb_partitions <= (wm->vtaps + 1))
2002 latency_tolerant_lines = 1;
2003 else
2004 latency_tolerant_lines = 2;
2005 }
2006
2007 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2008
2009 if (dce6_latency_watermark(wm) <= latency_hiding)
2010 return true;
2011 else
2012 return false;
2013}
2014
2015static void dce6_program_watermarks(struct radeon_device *rdev,
2016 struct radeon_crtc *radeon_crtc,
2017 u32 lb_size, u32 num_heads)
2018{
2019 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2020 struct dce6_wm_params wm_low, wm_high;
2021 u32 dram_channels;
43b3cd99
AD		 2022 u32 pixel_period;
2023 u32 line_time = 0;
2024 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2025 u32 priority_a_mark = 0, priority_b_mark = 0;
2026 u32 priority_a_cnt = PRIORITY_OFF;
2027 u32 priority_b_cnt = PRIORITY_OFF;
2028 u32 tmp, arb_control3;
2029 fixed20_12 a, b, c;
2030
2031 if (radeon_crtc->base.enabled && num_heads && mode) {
2032 pixel_period = 1000000 / (u32)mode->clock;
2033 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2034 priority_a_cnt = 0;
2035 priority_b_cnt = 0;
2036
ca7db22b 2037 if (rdev->family == CHIP_ARUBA)
c696e53f 2038 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2039 else
c696e53f
AD		 2040 dram_channels = si_get_number_of_dram_channels(rdev);
2041
2042 /* watermark for high clocks */
2043 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2044 wm_high.yclk =
2045 radeon_dpm_get_mclk(rdev, false) * 10;
2046 wm_high.sclk =
2047 radeon_dpm_get_sclk(rdev, false) * 10;
2048 } else {
2049 wm_high.yclk = rdev->pm.current_mclk * 10;
2050 wm_high.sclk = rdev->pm.current_sclk * 10;
2051 }
2052
2053 wm_high.disp_clk = mode->clock;
2054 wm_high.src_width = mode->crtc_hdisplay;
2055 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2056 wm_high.blank_time = line_time - wm_high.active_time;
2057 wm_high.interlaced = false;
2058 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2059 wm_high.interlaced = true;
2060 wm_high.vsc = radeon_crtc->vsc;
2061 wm_high.vtaps = 1;
2062 if (radeon_crtc->rmx_type != RMX_OFF)
2063 wm_high.vtaps = 2;
2064 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2065 wm_high.lb_size = lb_size;
2066 wm_high.dram_channels = dram_channels;
2067 wm_high.num_heads = num_heads;
2068
2069 /* watermark for low clocks */
2070 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2071 wm_low.yclk =
2072 radeon_dpm_get_mclk(rdev, true) * 10;
2073 wm_low.sclk =
2074 radeon_dpm_get_sclk(rdev, true) * 10;
2075 } else {
2076 wm_low.yclk = rdev->pm.current_mclk * 10;
2077 wm_low.sclk = rdev->pm.current_sclk * 10;
2078 }
2079
2080 wm_low.disp_clk = mode->clock;
2081 wm_low.src_width = mode->crtc_hdisplay;
2082 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2083 wm_low.blank_time = line_time - wm_low.active_time;
2084 wm_low.interlaced = false;
2085 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2086 wm_low.interlaced = true;
2087 wm_low.vsc = radeon_crtc->vsc;
2088 wm_low.vtaps = 1;
2089 if (radeon_crtc->rmx_type != RMX_OFF)
2090 wm_low.vtaps = 2;
2091 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2092 wm_low.lb_size = lb_size;
2093 wm_low.dram_channels = dram_channels;
2094 wm_low.num_heads = num_heads;
43b3cd99
AD		 2095
2096 /* set for high clocks */
c696e53f 2097 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2098 /* set for low clocks */
c696e53f 2099 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2100
2101 /* possibly force display priority to high */
2102 /* should really do this at mode validation time... */
c696e53f
AD		 2103 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2104 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2105 !dce6_check_latency_hiding(&wm_high) ||
2106 (rdev->disp_priority == 2)) {
2107 DRM_DEBUG_KMS("force priority to high\n");
2108 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2109 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2110 }
2111 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2112 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2113 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2114 (rdev->disp_priority == 2)) {
2115 DRM_DEBUG_KMS("force priority to high\n");
2116 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2117 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2118 }
2119
2120 a.full = dfixed_const(1000);
2121 b.full = dfixed_const(mode->clock);
2122 b.full = dfixed_div(b, a);
2123 c.full = dfixed_const(latency_watermark_a);
2124 c.full = dfixed_mul(c, b);
2125 c.full = dfixed_mul(c, radeon_crtc->hsc);
2126 c.full = dfixed_div(c, a);
2127 a.full = dfixed_const(16);
2128 c.full = dfixed_div(c, a);
2129 priority_a_mark = dfixed_trunc(c);
2130 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2131
2132 a.full = dfixed_const(1000);
2133 b.full = dfixed_const(mode->clock);
2134 b.full = dfixed_div(b, a);
2135 c.full = dfixed_const(latency_watermark_b);
2136 c.full = dfixed_mul(c, b);
2137 c.full = dfixed_mul(c, radeon_crtc->hsc);
2138 c.full = dfixed_div(c, a);
2139 a.full = dfixed_const(16);
2140 c.full = dfixed_div(c, a);
2141 priority_b_mark = dfixed_trunc(c);
2142 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2143 }
2144
2145 /* select wm A */
2146 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2147 tmp = arb_control3;
2148 tmp &= ~LATENCY_WATERMARK_MASK(3);
2149 tmp |= LATENCY_WATERMARK_MASK(1);
2150 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2151 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2152 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2153 LATENCY_HIGH_WATERMARK(line_time)));
2154 /* select wm B */
2155 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2156 tmp &= ~LATENCY_WATERMARK_MASK(3);
2157 tmp |= LATENCY_WATERMARK_MASK(2);
2158 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2159 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2160 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2161 LATENCY_HIGH_WATERMARK(line_time)));
2162 /* restore original selection */
2163 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2164
2165 /* write the priority marks */
2166 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2167 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2168
7178d2a6
AD		 2169 /* save values for DPM */
2170 radeon_crtc->line_time = line_time;
2171 radeon_crtc->wm_high = latency_watermark_a;
2172 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2173}
2174
2175void dce6_bandwidth_update(struct radeon_device *rdev)
2176{
2177 struct drm_display_mode *mode0 = NULL;
2178 struct drm_display_mode *mode1 = NULL;
2179 u32 num_heads = 0, lb_size;
2180 int i;
2181
2182 radeon_update_display_priority(rdev);
2183
2184 for (i = 0; i < rdev->num_crtc; i++) {
2185 if (rdev->mode_info.crtcs[i]->base.enabled)
2186 num_heads++;
2187 }
2188 for (i = 0; i < rdev->num_crtc; i += 2) {
2189 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2190 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2191 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2192 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2193 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2194 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2195 }
2196}
2197
0a96d72b
AD		 2198/*
2199 * Core functions
2200 */
0a96d72b
AD		 2201static void si_tiling_mode_table_init(struct radeon_device *rdev)
2202{
2203 const u32 num_tile_mode_states = 32;
2204 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2205
2206 switch (rdev->config.si.mem_row_size_in_kb) {
2207 case 1:
2208 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2209 break;
2210 case 2:
2211 default:
2212 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2213 break;
2214 case 4:
2215 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2216 break;
2217 }
2218
2219 if ((rdev->family == CHIP_TAHITI) ||
2220 (rdev->family == CHIP_PITCAIRN)) {
2221 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2222 switch (reg_offset) {
2223 case 0: /* non-AA compressed depth or any compressed stencil */
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2227 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2228 NUM_BANKS(ADDR_SURF_16_BANK) |
2229 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2230 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2231 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2232 break;
2233 case 1: /* 2xAA/4xAA compressed depth only */
2234 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2235 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2236 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2237 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2238 NUM_BANKS(ADDR_SURF_16_BANK) |
2239 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2240 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2241 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2242 break;
2243 case 2: /* 8xAA compressed depth only */
2244 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2245 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2246 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2247 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2248 NUM_BANKS(ADDR_SURF_16_BANK) |
2249 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2250 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2251 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2252 break;
2253 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2254 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2255 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2256 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2257 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2258 NUM_BANKS(ADDR_SURF_16_BANK) |
2259 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2260 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2261 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2262 break;
2263 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2264 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2265 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2266 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2267 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2268 NUM_BANKS(ADDR_SURF_16_BANK) |
2269 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2270 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2271 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2272 break;
2273 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2274 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2275 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2276 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2277 TILE_SPLIT(split_equal_to_row_size) |
2278 NUM_BANKS(ADDR_SURF_16_BANK) |
2279 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2280 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2281 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2282 break;
2283 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2285 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2286 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2287 TILE_SPLIT(split_equal_to_row_size) |
2288 NUM_BANKS(ADDR_SURF_16_BANK) |
2289 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2290 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2291 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2292 break;
2293 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2294 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2296 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2297 TILE_SPLIT(split_equal_to_row_size) |
2298 NUM_BANKS(ADDR_SURF_16_BANK) |
2299 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2300 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2301 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2302 break;
2303 case 8: /* 1D and 1D Array Surfaces */
2304 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2305 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2306 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2307 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2308 NUM_BANKS(ADDR_SURF_16_BANK) |
2309 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2310 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2311 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2312 break;
2313 case 9: /* Displayable maps. */
2314 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2315 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2316 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2317 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2318 NUM_BANKS(ADDR_SURF_16_BANK) |
2319 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2320 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2321 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2322 break;
2323 case 10: /* Display 8bpp. */
2324 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2325 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2326 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2327 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2328 NUM_BANKS(ADDR_SURF_16_BANK) |
2329 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2330 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2331 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2332 break;
2333 case 11: /* Display 16bpp. */
2334 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2335 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2336 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2337 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2338 NUM_BANKS(ADDR_SURF_16_BANK) |
2339 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2340 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2341 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2342 break;
2343 case 12: /* Display 32bpp. */
2344 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2345 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2346 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2347 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2348 NUM_BANKS(ADDR_SURF_16_BANK) |
2349 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2352 break;
2353 case 13: /* Thin. */
2354 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2355 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2356 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2357 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2358 NUM_BANKS(ADDR_SURF_16_BANK) |
2359 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2360 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2361 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2362 break;
2363 case 14: /* Thin 8 bpp. */
2364 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2365 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2366 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2367 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2368 NUM_BANKS(ADDR_SURF_16_BANK) |
2369 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2370 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2371 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2372 break;
2373 case 15: /* Thin 16 bpp. */
2374 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2375 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2376 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2377 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2378 NUM_BANKS(ADDR_SURF_16_BANK) |
2379 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2381 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2382 break;
2383 case 16: /* Thin 32 bpp. */
2384 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2385 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2386 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2387 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2388 NUM_BANKS(ADDR_SURF_16_BANK) |
2389 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2390 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2391 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2392 break;
2393 case 17: /* Thin 64 bpp. */
2394 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2395 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2396 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2397 TILE_SPLIT(split_equal_to_row_size) |
2398 NUM_BANKS(ADDR_SURF_16_BANK) |
2399 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2400 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2401 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2402 break;
2403 case 21: /* 8 bpp PRT. */
2404 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2406 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2407 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2408 NUM_BANKS(ADDR_SURF_16_BANK) |
2409 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2412 break;
2413 case 22: /* 16 bpp PRT */
2414 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2415 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2416 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2417 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2418 NUM_BANKS(ADDR_SURF_16_BANK) |
2419 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2420 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2421 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2422 break;
2423 case 23: /* 32 bpp PRT */
2424 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2425 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2426 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2427 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2428 NUM_BANKS(ADDR_SURF_16_BANK) |
2429 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2430 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2431 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2432 break;
2433 case 24: /* 64 bpp PRT */
2434 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2435 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2436 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2437 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2438 NUM_BANKS(ADDR_SURF_16_BANK) |
2439 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2440 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2441 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2442 break;
2443 case 25: /* 128 bpp PRT */
2444 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2445 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2446 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2447 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2448 NUM_BANKS(ADDR_SURF_8_BANK) |
2449 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2452 break;
2453 default:
2454 gb_tile_moden = 0;
2455 break;
2456 }
64d7b8be 2457 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2458 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2459 }
d0ae7fcc 2460 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2461 (rdev->family == CHIP_OLAND) ||
2462 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2463 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2464 switch (reg_offset) {
2465 case 0: /* non-AA compressed depth or any compressed stencil */
2466 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2467 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2468 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2469 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2470 NUM_BANKS(ADDR_SURF_16_BANK) |
2471 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2472 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2473 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2474 break;
2475 case 1: /* 2xAA/4xAA compressed depth only */
2476 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2477 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2478 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2479 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2480 NUM_BANKS(ADDR_SURF_16_BANK) |
2481 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2484 break;
2485 case 2: /* 8xAA compressed depth only */
2486 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2487 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2488 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2489 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2490 NUM_BANKS(ADDR_SURF_16_BANK) |
2491 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2493 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2494 break;
2495 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2496 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2497 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2498 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2499 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2500 NUM_BANKS(ADDR_SURF_16_BANK) |
2501 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2502 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2503 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2504 break;
2505 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2506 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2507 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2508 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2509 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2510 NUM_BANKS(ADDR_SURF_16_BANK) |
2511 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2514 break;
2515 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2516 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2517 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2518 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2519 TILE_SPLIT(split_equal_to_row_size) |
2520 NUM_BANKS(ADDR_SURF_16_BANK) |
2521 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2522 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2523 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2524 break;
2525 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2526 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2527 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2528 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2529 TILE_SPLIT(split_equal_to_row_size) |
2530 NUM_BANKS(ADDR_SURF_16_BANK) |
2531 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2532 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2533 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2534 break;
2535 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2536 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2539 TILE_SPLIT(split_equal_to_row_size) |
2540 NUM_BANKS(ADDR_SURF_16_BANK) |
2541 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2542 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2543 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2544 break;
2545 case 8: /* 1D and 1D Array Surfaces */
2546 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2547 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2548 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2549 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2550 NUM_BANKS(ADDR_SURF_16_BANK) |
2551 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2552 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2553 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2554 break;
2555 case 9: /* Displayable maps. */
2556 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2557 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2558 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2559 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2560 NUM_BANKS(ADDR_SURF_16_BANK) |
2561 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2562 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2563 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2564 break;
2565 case 10: /* Display 8bpp. */
2566 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2567 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2568 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2569 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2570 NUM_BANKS(ADDR_SURF_16_BANK) |
2571 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2572 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2573 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2574 break;
2575 case 11: /* Display 16bpp. */
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2577 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2578 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2579 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2580 NUM_BANKS(ADDR_SURF_16_BANK) |
2581 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2582 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2583 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2584 break;
2585 case 12: /* Display 32bpp. */
2586 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2587 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2588 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2589 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2590 NUM_BANKS(ADDR_SURF_16_BANK) |
2591 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2593 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2594 break;
2595 case 13: /* Thin. */
2596 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2597 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2598 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2599 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2600 NUM_BANKS(ADDR_SURF_16_BANK) |
2601 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2602 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2603 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2604 break;
2605 case 14: /* Thin 8 bpp. */
2606 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2607 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2608 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2609 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2610 NUM_BANKS(ADDR_SURF_16_BANK) |
2611 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2612 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2613 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2614 break;
2615 case 15: /* Thin 16 bpp. */
2616 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2617 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2618 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2619 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2620 NUM_BANKS(ADDR_SURF_16_BANK) |
2621 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2622 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2623 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2624 break;
2625 case 16: /* Thin 32 bpp. */
2626 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2627 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2628 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2629 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2630 NUM_BANKS(ADDR_SURF_16_BANK) |
2631 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2633 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2634 break;
2635 case 17: /* Thin 64 bpp. */
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2637 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2638 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2639 TILE_SPLIT(split_equal_to_row_size) |
2640 NUM_BANKS(ADDR_SURF_16_BANK) |
2641 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2642 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2643 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2644 break;
2645 case 21: /* 8 bpp PRT. */
2646 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2647 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2648 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2649 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2650 NUM_BANKS(ADDR_SURF_16_BANK) |
2651 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2652 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2653 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2654 break;
2655 case 22: /* 16 bpp PRT */
2656 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2657 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2658 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2659 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2660 NUM_BANKS(ADDR_SURF_16_BANK) |
2661 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2662 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2663 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2664 break;
2665 case 23: /* 32 bpp PRT */
2666 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2667 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2668 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2669 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2670 NUM_BANKS(ADDR_SURF_16_BANK) |
2671 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2672 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2673 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2674 break;
2675 case 24: /* 64 bpp PRT */
2676 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2677 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2678 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2679 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2680 NUM_BANKS(ADDR_SURF_16_BANK) |
2681 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2682 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2683 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2684 break;
2685 case 25: /* 128 bpp PRT */
2686 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2687 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2688 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2689 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2690 NUM_BANKS(ADDR_SURF_8_BANK) |
2691 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2692 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2693 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2694 break;
2695 default:
2696 gb_tile_moden = 0;
2697 break;
2698 }
64d7b8be 2699 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2700 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2701 }
2702 } else
2703 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2704}
2705
1a8ca750
AD		 2706static void si_select_se_sh(struct radeon_device *rdev,
2707 u32 se_num, u32 sh_num)
2708{
2709 u32 data = INSTANCE_BROADCAST_WRITES;
2710
2711 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2712 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2713 else if (se_num == 0xffffffff)
2714 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2715 else if (sh_num == 0xffffffff)
2716 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2717 else
2718 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2719 WREG32(GRBM_GFX_INDEX, data);
2720}
2721
2722static u32 si_create_bitmask(u32 bit_width)
2723{
2724 u32 i, mask = 0;
2725
2726 for (i = 0; i < bit_width; i++) {
2727 mask <<= 1;
2728 mask |= 1;
2729 }
2730 return mask;
2731}
2732
2733static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2734{
2735 u32 data, mask;
2736
2737 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2738 if (data & 1)
2739 data &= INACTIVE_CUS_MASK;
2740 else
2741 data = 0;
2742 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2743
2744 data >>= INACTIVE_CUS_SHIFT;
2745
2746 mask = si_create_bitmask(cu_per_sh);
2747
2748 return ~data & mask;
2749}
2750
2751static void si_setup_spi(struct radeon_device *rdev,
2752 u32 se_num, u32 sh_per_se,
2753 u32 cu_per_sh)
2754{
2755 int i, j, k;
2756 u32 data, mask, active_cu;
2757
2758 for (i = 0; i < se_num; i++) {
2759 for (j = 0; j < sh_per_se; j++) {
2760 si_select_se_sh(rdev, i, j);
2761 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2762 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2763
2764 mask = 1;
2765 for (k = 0; k < 16; k++) {
2766 mask <<= k;
2767 if (active_cu & mask) {
2768 data &= ~mask;
2769 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2770 break;
2771 }
2772 }
2773 }
2774 }
2775 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2776}
2777
2778static u32 si_get_rb_disabled(struct radeon_device *rdev,
2779 u32 max_rb_num, u32 se_num,
2780 u32 sh_per_se)
2781{
2782 u32 data, mask;
2783
2784 data = RREG32(CC_RB_BACKEND_DISABLE);
2785 if (data & 1)
2786 data &= BACKEND_DISABLE_MASK;
2787 else
2788 data = 0;
2789 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2790
2791 data >>= BACKEND_DISABLE_SHIFT;
2792
2793 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2794
2795 return data & mask;
2796}
2797
2798static void si_setup_rb(struct radeon_device *rdev,
2799 u32 se_num, u32 sh_per_se,
2800 u32 max_rb_num)
2801{
2802 int i, j;
2803 u32 data, mask;
2804 u32 disabled_rbs = 0;
2805 u32 enabled_rbs = 0;
2806
2807 for (i = 0; i < se_num; i++) {
2808 for (j = 0; j < sh_per_se; j++) {
2809 si_select_se_sh(rdev, i, j);
2810 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2811 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2812 }
2813 }
2814 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2815
2816 mask = 1;
2817 for (i = 0; i < max_rb_num; i++) {
2818 if (!(disabled_rbs & mask))
2819 enabled_rbs |= mask;
2820 mask <<= 1;
2821 }
2822
2823 for (i = 0; i < se_num; i++) {
2824 si_select_se_sh(rdev, i, 0xffffffff);
2825 data = 0;
2826 for (j = 0; j < sh_per_se; j++) {
2827 switch (enabled_rbs & 3) {
2828 case 1:
2829 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2830 break;
2831 case 2:
2832 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2833 break;
2834 case 3:
2835 default:
2836 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2837 break;
2838 }
2839 enabled_rbs >>= 2;
2840 }
2841 WREG32(PA_SC_RASTER_CONFIG, data);
2842 }
2843 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2844}
2845
0a96d72b
AD		 2846static void si_gpu_init(struct radeon_device *rdev)
2847{
0a96d72b
AD		 2848 u32 gb_addr_config = 0;
2849 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2850 u32 sx_debug_1;
0a96d72b
AD		 2851 u32 hdp_host_path_cntl;
2852 u32 tmp;
2853 int i, j;
2854
2855 switch (rdev->family) {
2856 case CHIP_TAHITI:
2857 rdev->config.si.max_shader_engines = 2;
0a96d72b 2858 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2859 rdev->config.si.max_cu_per_sh = 8;
2860 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2861 rdev->config.si.max_backends_per_se = 4;
2862 rdev->config.si.max_texture_channel_caches = 12;
2863 rdev->config.si.max_gprs = 256;
2864 rdev->config.si.max_gs_threads = 32;
2865 rdev->config.si.max_hw_contexts = 8;
2866
2867 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2868 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2869 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2870 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2871 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2872 break;
2873 case CHIP_PITCAIRN:
2874 rdev->config.si.max_shader_engines = 2;
0a96d72b 2875 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2876 rdev->config.si.max_cu_per_sh = 5;
2877 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2878 rdev->config.si.max_backends_per_se = 4;
2879 rdev->config.si.max_texture_channel_caches = 8;
2880 rdev->config.si.max_gprs = 256;
2881 rdev->config.si.max_gs_threads = 32;
2882 rdev->config.si.max_hw_contexts = 8;
2883
2884 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2885 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2886 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2887 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2888 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2889 break;
2890 case CHIP_VERDE:
2891 default:
2892 rdev->config.si.max_shader_engines = 1;
0a96d72b 2893 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2894 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2895 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2896 rdev->config.si.max_backends_per_se = 4;
2897 rdev->config.si.max_texture_channel_caches = 4;
2898 rdev->config.si.max_gprs = 256;
2899 rdev->config.si.max_gs_threads = 32;
2900 rdev->config.si.max_hw_contexts = 8;
2901
d0ae7fcc
AD		 2902 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2903 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2904 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2905 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2906 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2907 break;
2908 case CHIP_OLAND:
2909 rdev->config.si.max_shader_engines = 1;
2910 rdev->config.si.max_tile_pipes = 4;
2911 rdev->config.si.max_cu_per_sh = 6;
2912 rdev->config.si.max_sh_per_se = 1;
2913 rdev->config.si.max_backends_per_se = 2;
2914 rdev->config.si.max_texture_channel_caches = 4;
2915 rdev->config.si.max_gprs = 256;
2916 rdev->config.si.max_gs_threads = 16;
2917 rdev->config.si.max_hw_contexts = 8;
2918
0a96d72b
AD		 2919 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2920 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2921 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2922 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2923 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2924 break;
8b02859d
AD		 2925 case CHIP_HAINAN:
2926 rdev->config.si.max_shader_engines = 1;
2927 rdev->config.si.max_tile_pipes = 4;
2928 rdev->config.si.max_cu_per_sh = 5;
2929 rdev->config.si.max_sh_per_se = 1;
2930 rdev->config.si.max_backends_per_se = 1;
2931 rdev->config.si.max_texture_channel_caches = 2;
2932 rdev->config.si.max_gprs = 256;
2933 rdev->config.si.max_gs_threads = 16;
2934 rdev->config.si.max_hw_contexts = 8;
2935
2936 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2937 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2938 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2939 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2940 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2941 break;
0a96d72b
AD		 2942 }
2943
2944 /* Initialize HDP */
2945 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2946 WREG32((0x2c14 + j), 0x00000000);
2947 WREG32((0x2c18 + j), 0x00000000);
2948 WREG32((0x2c1c + j), 0x00000000);
2949 WREG32((0x2c20 + j), 0x00000000);
2950 WREG32((0x2c24 + j), 0x00000000);
2951 }
2952
2953 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2954
2955 evergreen_fix_pci_max_read_req_size(rdev);
2956
2957 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2958
2959 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2960 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2961
0a96d72b 2962 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 2963 rdev->config.si.mem_max_burst_length_bytes = 256;
2964 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2965 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2966 if (rdev->config.si.mem_row_size_in_kb > 4)
2967 rdev->config.si.mem_row_size_in_kb = 4;
2968 /* XXX use MC settings? */
2969 rdev->config.si.shader_engine_tile_size = 32;
2970 rdev->config.si.num_gpus = 1;
2971 rdev->config.si.multi_gpu_tile_size = 64;
2972
1a8ca750
AD		 2973 /* fix up row size */
2974 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 2975 switch (rdev->config.si.mem_row_size_in_kb) {
2976 case 1:
2977 default:
2978 gb_addr_config |= ROW_SIZE(0);
2979 break;
2980 case 2:
2981 gb_addr_config |= ROW_SIZE(1);
2982 break;
2983 case 4:
2984 gb_addr_config |= ROW_SIZE(2);
2985 break;
2986 }
2987
0a96d72b
AD		 2988 /* setup tiling info dword. gb_addr_config is not adequate since it does
2989 * not have bank info, so create a custom tiling dword.
2990 * bits 3:0 num_pipes
2991 * bits 7:4 num_banks
2992 * bits 11:8 group_size
2993 * bits 15:12 row_size
2994 */
2995 rdev->config.si.tile_config = 0;
2996 switch (rdev->config.si.num_tile_pipes) {
2997 case 1:
2998 rdev->config.si.tile_config |= (0 << 0);
2999 break;
3000 case 2:
3001 rdev->config.si.tile_config |= (1 << 0);
3002 break;
3003 case 4:
3004 rdev->config.si.tile_config |= (2 << 0);
3005 break;
3006 case 8:
3007 default:
3008 /* XXX what about 12? */
3009 rdev->config.si.tile_config |= (3 << 0);
3010 break;
dca571a6
CK		 3011 }
3012 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3013 case 0: /* four banks */
1a8ca750 3014 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3015 break;
3016 case 1: /* eight banks */
3017 rdev->config.si.tile_config |= 1 << 4;
3018 break;
3019 case 2: /* sixteen banks */
3020 default:
3021 rdev->config.si.tile_config |= 2 << 4;
3022 break;
3023 }
0a96d72b
AD		 3024 rdev->config.si.tile_config |=
3025 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3026 rdev->config.si.tile_config |=
3027 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3028
0a96d72b
AD		 3029 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3030 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3031 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3032 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3033 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3034 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3035 if (rdev->has_uvd) {
3036 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3037 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3038 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3039 }
0a96d72b 3040
1a8ca750 3041 si_tiling_mode_table_init(rdev);
0a96d72b 3042
1a8ca750
AD		 3043 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3044 rdev->config.si.max_sh_per_se,
3045 rdev->config.si.max_backends_per_se);
0a96d72b 3046
1a8ca750
AD		 3047 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3048 rdev->config.si.max_sh_per_se,
3049 rdev->config.si.max_cu_per_sh);
0a96d72b 3050
0a96d72b
AD		 3051
3052 /* set HW defaults for 3D engine */
3053 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3054 ROQ_IB2_START(0x2b)));
3055 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3056
3057 sx_debug_1 = RREG32(SX_DEBUG_1);
3058 WREG32(SX_DEBUG_1, sx_debug_1);
3059
3060 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3061
3062 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3063 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3064 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3065 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3066
3067 WREG32(VGT_NUM_INSTANCES, 1);
3068
3069 WREG32(CP_PERFMON_CNTL, 0);
3070
3071 WREG32(SQ_CONFIG, 0);
3072
3073 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3074 FORCE_EOV_MAX_REZ_CNT(255)));
3075
3076 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3077 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3078
3079 WREG32(VGT_GS_VERTEX_REUSE, 16);
3080 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3081
3082 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3083 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3084 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3085 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3086 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3087 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3088 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3089 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3090
3091 tmp = RREG32(HDP_MISC_CNTL);
3092 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3093 WREG32(HDP_MISC_CNTL, tmp);
3094
3095 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3096 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3097
3098 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3099
3100 udelay(50);
3101}
c476dde2 3102
2ece2e8b
AD		 3103/*
3104 * GPU scratch registers helpers function.
3105 */
3106static void si_scratch_init(struct radeon_device *rdev)
3107{
3108 int i;
3109
3110 rdev->scratch.num_reg = 7;
3111 rdev->scratch.reg_base = SCRATCH_REG0;
3112 for (i = 0; i < rdev->scratch.num_reg; i++) {
3113 rdev->scratch.free[i] = true;
3114 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3115 }
3116}
3117
3118void si_fence_ring_emit(struct radeon_device *rdev,
3119 struct radeon_fence *fence)
3120{
3121 struct radeon_ring *ring = &rdev->ring[fence->ring];
3122 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3123
3124 /* flush read cache over gart */
3125 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3126 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3127 radeon_ring_write(ring, 0);
3128 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3129 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3130 PACKET3_TC_ACTION_ENA |
3131 PACKET3_SH_KCACHE_ACTION_ENA |
3132 PACKET3_SH_ICACHE_ACTION_ENA);
3133 radeon_ring_write(ring, 0xFFFFFFFF);
3134 radeon_ring_write(ring, 0);
3135 radeon_ring_write(ring, 10); /* poll interval */
3136 /* EVENT_WRITE_EOP - flush caches, send int */
3137 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3138 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3139 radeon_ring_write(ring, addr & 0xffffffff);
3140 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3141 radeon_ring_write(ring, fence->seq);
3142 radeon_ring_write(ring, 0);
3143}
3144
3145/*
3146 * IB stuff
3147 */
3148void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3149{
876dc9f3 3150 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 3151 u32 header;
3152
a85a7da4
AD		 3153 if (ib->is_const_ib) {
3154 /* set switch buffer packet before const IB */
3155 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3156 radeon_ring_write(ring, 0);
45df6803 3157
2ece2e8b 3158 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3159 } else {
89d35807 3160 u32 next_rptr;
a85a7da4 3161 if (ring->rptr_save_reg) {
89d35807 3162 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3163 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3164 radeon_ring_write(ring, ((ring->rptr_save_reg -
3165 PACKET3_SET_CONFIG_REG_START) >> 2));
3166 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3167 } else if (rdev->wb.enabled) {
3168 next_rptr = ring->wptr + 5 + 4 + 8;
3169 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3170 radeon_ring_write(ring, (1 << 8));
3171 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3172 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3173 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3174 }
3175
2ece2e8b 3176 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3177 }
2ece2e8b
AD		 3178
3179 radeon_ring_write(ring, header);
3180 radeon_ring_write(ring,
3181#ifdef __BIG_ENDIAN
3182 (2 << 0) |
3183#endif
3184 (ib->gpu_addr & 0xFFFFFFFC));
3185 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 3186 radeon_ring_write(ring, ib->length_dw |
3187 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 3188
a85a7da4
AD		 3189 if (!ib->is_const_ib) {
3190 /* flush read cache over gart for this vmid */
3191 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3192 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 3193 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 3194 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3195 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3196 PACKET3_TC_ACTION_ENA |
3197 PACKET3_SH_KCACHE_ACTION_ENA |
3198 PACKET3_SH_ICACHE_ACTION_ENA);
3199 radeon_ring_write(ring, 0xFFFFFFFF);
3200 radeon_ring_write(ring, 0);
3201 radeon_ring_write(ring, 10); /* poll interval */
3202 }
2ece2e8b
AD		 3203}
3204
48c0c902
AD		 3205/*
3206 * CP.
3207 */
3208static void si_cp_enable(struct radeon_device *rdev, bool enable)
3209{
3210 if (enable)
3211 WREG32(CP_ME_CNTL, 0);
3212 else {
3213 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3214 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3215 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3216 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3217 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3218 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3219 }
3220 udelay(50);
3221}
3222
3223static int si_cp_load_microcode(struct radeon_device *rdev)
3224{
3225 const __be32 *fw_data;
3226 int i;
3227
3228 if (!rdev->me_fw || !rdev->pfp_fw)
3229 return -EINVAL;
3230
3231 si_cp_enable(rdev, false);
3232
3233 /* PFP */
3234 fw_data = (const __be32 *)rdev->pfp_fw->data;
3235 WREG32(CP_PFP_UCODE_ADDR, 0);
3236 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3237 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3238 WREG32(CP_PFP_UCODE_ADDR, 0);
3239
3240 /* CE */
3241 fw_data = (const __be32 *)rdev->ce_fw->data;
3242 WREG32(CP_CE_UCODE_ADDR, 0);
3243 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3244 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3245 WREG32(CP_CE_UCODE_ADDR, 0);
3246
3247 /* ME */
3248 fw_data = (const __be32 *)rdev->me_fw->data;
3249 WREG32(CP_ME_RAM_WADDR, 0);
3250 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3251 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3252 WREG32(CP_ME_RAM_WADDR, 0);
3253
3254 WREG32(CP_PFP_UCODE_ADDR, 0);
3255 WREG32(CP_CE_UCODE_ADDR, 0);
3256 WREG32(CP_ME_RAM_WADDR, 0);
3257 WREG32(CP_ME_RAM_RADDR, 0);
3258 return 0;
3259}
3260
3261static int si_cp_start(struct radeon_device *rdev)
3262{
3263 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3264 int r, i;
3265
3266 r = radeon_ring_lock(rdev, ring, 7 + 4);
3267 if (r) {
3268 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3269 return r;
3270 }
3271 /* init the CP */
3272 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3273 radeon_ring_write(ring, 0x1);
3274 radeon_ring_write(ring, 0x0);
3275 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3276 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3277 radeon_ring_write(ring, 0);
3278 radeon_ring_write(ring, 0);
3279
3280 /* init the CE partitions */
3281 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3282 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3283 radeon_ring_write(ring, 0xc000);
3284 radeon_ring_write(ring, 0xe000);
3285 radeon_ring_unlock_commit(rdev, ring);
3286
3287 si_cp_enable(rdev, true);
3288
3289 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3290 if (r) {
3291 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3292 return r;
3293 }
3294
3295 /* setup clear context state */
3296 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3297 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3298
3299 for (i = 0; i < si_default_size; i++)
3300 radeon_ring_write(ring, si_default_state[i]);
3301
3302 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3303 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3304
3305 /* set clear context state */
3306 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3307 radeon_ring_write(ring, 0);
3308
3309 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3310 radeon_ring_write(ring, 0x00000316);
3311 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3312 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3313
3314 radeon_ring_unlock_commit(rdev, ring);
3315
3316 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3317 ring = &rdev->ring[i];
3318 r = radeon_ring_lock(rdev, ring, 2);
3319
3320 /* clear the compute context state */
3321 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3322 radeon_ring_write(ring, 0);
3323
3324 radeon_ring_unlock_commit(rdev, ring);
3325 }
3326
3327 return 0;
3328}
3329
3330static void si_cp_fini(struct radeon_device *rdev)
3331{
45df6803 3332 struct radeon_ring *ring;
48c0c902 3333 si_cp_enable(rdev, false);
45df6803
CK		 3334
3335 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3336 radeon_ring_fini(rdev, ring);
3337 radeon_scratch_free(rdev, ring->rptr_save_reg);
3338
3339 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3340 radeon_ring_fini(rdev, ring);
3341 radeon_scratch_free(rdev, ring->rptr_save_reg);
3342
3343 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3344 radeon_ring_fini(rdev, ring);
3345 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3346}
3347
3348static int si_cp_resume(struct radeon_device *rdev)
3349{
3350 struct radeon_ring *ring;
3351 u32 tmp;
3352 u32 rb_bufsz;
3353 int r;
3354
3355 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3356 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3357 SOFT_RESET_PA |
3358 SOFT_RESET_VGT |
3359 SOFT_RESET_SPI |
3360 SOFT_RESET_SX));
3361 RREG32(GRBM_SOFT_RESET);
3362 mdelay(15);
3363 WREG32(GRBM_SOFT_RESET, 0);
3364 RREG32(GRBM_SOFT_RESET);
3365
3366 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3367 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3368
3369 /* Set the write pointer delay */
3370 WREG32(CP_RB_WPTR_DELAY, 0);
3371
3372 WREG32(CP_DEBUG, 0);
3373 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3374
3375 /* ring 0 - compute and gfx */
3376 /* Set ring buffer size */
3377 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3378 rb_bufsz = drm_order(ring->ring_size / 8);
3379 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3380#ifdef __BIG_ENDIAN
3381 tmp |= BUF_SWAP_32BIT;
3382#endif
3383 WREG32(CP_RB0_CNTL, tmp);
3384
3385 /* Initialize the ring buffer's read and write pointers */
3386 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3387 ring->wptr = 0;
3388 WREG32(CP_RB0_WPTR, ring->wptr);
3389
48fc7f7e 3390 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3391 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3392 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3393
3394 if (rdev->wb.enabled)
3395 WREG32(SCRATCH_UMSK, 0xff);
3396 else {
3397 tmp |= RB_NO_UPDATE;
3398 WREG32(SCRATCH_UMSK, 0);
3399 }
3400
3401 mdelay(1);
3402 WREG32(CP_RB0_CNTL, tmp);
3403
3404 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3405
3406 ring->rptr = RREG32(CP_RB0_RPTR);
3407
3408 /* ring1 - compute only */
3409 /* Set ring buffer size */
3410 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3411 rb_bufsz = drm_order(ring->ring_size / 8);
3412 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3413#ifdef __BIG_ENDIAN
3414 tmp |= BUF_SWAP_32BIT;
3415#endif
3416 WREG32(CP_RB1_CNTL, tmp);
3417
3418 /* Initialize the ring buffer's read and write pointers */
3419 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3420 ring->wptr = 0;
3421 WREG32(CP_RB1_WPTR, ring->wptr);
3422
48fc7f7e 3423 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3424 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3425 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3426
3427 mdelay(1);
3428 WREG32(CP_RB1_CNTL, tmp);
3429
3430 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3431
3432 ring->rptr = RREG32(CP_RB1_RPTR);
3433
3434 /* ring2 - compute only */
3435 /* Set ring buffer size */
3436 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3437 rb_bufsz = drm_order(ring->ring_size / 8);
3438 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3439#ifdef __BIG_ENDIAN
3440 tmp |= BUF_SWAP_32BIT;
3441#endif
3442 WREG32(CP_RB2_CNTL, tmp);
3443
3444 /* Initialize the ring buffer's read and write pointers */
3445 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3446 ring->wptr = 0;
3447 WREG32(CP_RB2_WPTR, ring->wptr);
3448
48fc7f7e 3449 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3450 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3451 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3452
3453 mdelay(1);
3454 WREG32(CP_RB2_CNTL, tmp);
3455
3456 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3457
3458 ring->rptr = RREG32(CP_RB2_RPTR);
3459
3460 /* start the rings */
3461 si_cp_start(rdev);
3462 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3463 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3464 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3465 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3466 if (r) {
3467 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3468 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3469 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3470 return r;
3471 }
3472 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3473 if (r) {
3474 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3475 }
3476 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3477 if (r) {
3478 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3479 }
3480
3481 return 0;
3482}
3483
014bb209 3484static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3485{
014bb209 3486 u32 reset_mask = 0;
1c534671 3487 u32 tmp;
06bc6df0 3488
014bb209
AD		 3489 /* GRBM_STATUS */
3490 tmp = RREG32(GRBM_STATUS);
3491 if (tmp & (PA_BUSY | SC_BUSY |
3492 BCI_BUSY | SX_BUSY |
3493 TA_BUSY | VGT_BUSY |
3494 DB_BUSY | CB_BUSY |
3495 GDS_BUSY | SPI_BUSY |
3496 IA_BUSY | IA_BUSY_NO_DMA))
3497 reset_mask |= RADEON_RESET_GFX;
3498
3499 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3500 CP_BUSY | CP_COHERENCY_BUSY))
3501 reset_mask |= RADEON_RESET_CP;
3502
3503 if (tmp & GRBM_EE_BUSY)
3504 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3505
3506 /* GRBM_STATUS2 */
3507 tmp = RREG32(GRBM_STATUS2);
3508 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3509 reset_mask |= RADEON_RESET_RLC;
3510
3511 /* DMA_STATUS_REG 0 */
3512 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3513 if (!(tmp & DMA_IDLE))
3514 reset_mask |= RADEON_RESET_DMA;
3515
3516 /* DMA_STATUS_REG 1 */
3517 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3518 if (!(tmp & DMA_IDLE))
3519 reset_mask |= RADEON_RESET_DMA1;
3520
3521 /* SRBM_STATUS2 */
3522 tmp = RREG32(SRBM_STATUS2);
3523 if (tmp & DMA_BUSY)
3524 reset_mask |= RADEON_RESET_DMA;
3525
3526 if (tmp & DMA1_BUSY)
3527 reset_mask |= RADEON_RESET_DMA1;
3528
3529 /* SRBM_STATUS */
3530 tmp = RREG32(SRBM_STATUS);
3531
3532 if (tmp & IH_BUSY)
3533 reset_mask |= RADEON_RESET_IH;
3534
3535 if (tmp & SEM_BUSY)
3536 reset_mask |= RADEON_RESET_SEM;
3537
3538 if (tmp & GRBM_RQ_PENDING)
3539 reset_mask |= RADEON_RESET_GRBM;
3540
3541 if (tmp & VMC_BUSY)
3542 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3543
014bb209
AD		 3544 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3545 MCC_BUSY | MCD_BUSY))
3546 reset_mask |= RADEON_RESET_MC;
3547
3548 if (evergreen_is_display_hung(rdev))
3549 reset_mask |= RADEON_RESET_DISPLAY;
3550
3551 /* VM_L2_STATUS */
3552 tmp = RREG32(VM_L2_STATUS);
3553 if (tmp & L2_BUSY)
3554 reset_mask |= RADEON_RESET_VMC;
3555
d808fc88
AD		 3556 /* Skip MC reset as it's mostly likely not hung, just busy */
3557 if (reset_mask & RADEON_RESET_MC) {
3558 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3559 reset_mask &= ~RADEON_RESET_MC;
3560 }
3561
014bb209
AD		 3562 return reset_mask;
3563}
3564
3565static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3566{
3567 struct evergreen_mc_save save;
3568 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3569 u32 tmp;
19fc42ed 3570
06bc6df0 3571 if (reset_mask == 0)
014bb209 3572 return;
06bc6df0
AD		 3573
3574 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3575
1c534671 3576 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3577 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3578 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3579 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3580 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3581
1c534671
AD		 3582 /* Disable CP parsing/prefetching */
3583 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3584
3585 if (reset_mask & RADEON_RESET_DMA) {
3586 /* dma0 */
3587 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3588 tmp &= ~DMA_RB_ENABLE;
3589 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3590 }
3591 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3592 /* dma1 */
3593 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3594 tmp &= ~DMA_RB_ENABLE;
3595 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3596 }
3597
f770d78a
AD		 3598 udelay(50);
3599
3600 evergreen_mc_stop(rdev, &save);
3601 if (evergreen_mc_wait_for_idle(rdev)) {
3602 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3603 }
3604
1c534671
AD		 3605 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3606 grbm_soft_reset = SOFT_RESET_CB |
3607 SOFT_RESET_DB |
3608 SOFT_RESET_GDS |
3609 SOFT_RESET_PA |
3610 SOFT_RESET_SC |
3611 SOFT_RESET_BCI |
3612 SOFT_RESET_SPI |
3613 SOFT_RESET_SX |
3614 SOFT_RESET_TC |
3615 SOFT_RESET_TA |
3616 SOFT_RESET_VGT |
3617 SOFT_RESET_IA;
3618 }
3619
3620 if (reset_mask & RADEON_RESET_CP) {
3621 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3622
3623 srbm_soft_reset |= SOFT_RESET_GRBM;
3624 }
06bc6df0
AD		 3625
3626 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3627 srbm_soft_reset |= SOFT_RESET_DMA;
3628
3629 if (reset_mask & RADEON_RESET_DMA1)
3630 srbm_soft_reset |= SOFT_RESET_DMA1;
3631
3632 if (reset_mask & RADEON_RESET_DISPLAY)
3633 srbm_soft_reset |= SOFT_RESET_DC;
3634
3635 if (reset_mask & RADEON_RESET_RLC)
3636 grbm_soft_reset |= SOFT_RESET_RLC;
3637
3638 if (reset_mask & RADEON_RESET_SEM)
3639 srbm_soft_reset |= SOFT_RESET_SEM;
3640
3641 if (reset_mask & RADEON_RESET_IH)
3642 srbm_soft_reset |= SOFT_RESET_IH;
3643
3644 if (reset_mask & RADEON_RESET_GRBM)
3645 srbm_soft_reset |= SOFT_RESET_GRBM;
3646
3647 if (reset_mask & RADEON_RESET_VMC)
3648 srbm_soft_reset |= SOFT_RESET_VMC;
3649
3650 if (reset_mask & RADEON_RESET_MC)
3651 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3652
3653 if (grbm_soft_reset) {
3654 tmp = RREG32(GRBM_SOFT_RESET);
3655 tmp |= grbm_soft_reset;
3656 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3657 WREG32(GRBM_SOFT_RESET, tmp);
3658 tmp = RREG32(GRBM_SOFT_RESET);
3659
3660 udelay(50);
3661
3662 tmp &= ~grbm_soft_reset;
3663 WREG32(GRBM_SOFT_RESET, tmp);
3664 tmp = RREG32(GRBM_SOFT_RESET);
3665 }
3666
3667 if (srbm_soft_reset) {
3668 tmp = RREG32(SRBM_SOFT_RESET);
3669 tmp |= srbm_soft_reset;
3670 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3671 WREG32(SRBM_SOFT_RESET, tmp);
3672 tmp = RREG32(SRBM_SOFT_RESET);
3673
3674 udelay(50);
3675
3676 tmp &= ~srbm_soft_reset;
3677 WREG32(SRBM_SOFT_RESET, tmp);
3678 tmp = RREG32(SRBM_SOFT_RESET);
3679 }
06bc6df0
AD		 3680
3681 /* Wait a little for things to settle down */
3682 udelay(50);
3683
c476dde2 3684 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3685 udelay(50);
3686
1c534671 3687 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3688}
3689
3690int si_asic_reset(struct radeon_device *rdev)
3691{
014bb209
AD		 3692 u32 reset_mask;
3693
3694 reset_mask = si_gpu_check_soft_reset(rdev);
3695
3696 if (reset_mask)
3697 r600_set_bios_scratch_engine_hung(rdev, true);
3698
3699 si_gpu_soft_reset(rdev, reset_mask);
3700
3701 reset_mask = si_gpu_check_soft_reset(rdev);
3702
3703 if (!reset_mask)
3704 r600_set_bios_scratch_engine_hung(rdev, false);
3705
3706 return 0;
c476dde2
AD		 3707}
3708
123bc183
AD		 3709/**
3710 * si_gfx_is_lockup - Check if the GFX engine is locked up
3711 *
3712 * @rdev: radeon_device pointer
3713 * @ring: radeon_ring structure holding ring information
3714 *
3715 * Check if the GFX engine is locked up.
3716 * Returns true if the engine appears to be locked up, false if not.
3717 */
3718bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3719{
3720 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3721
3722 if (!(reset_mask & (RADEON_RESET_GFX |
3723 RADEON_RESET_COMPUTE |
3724 RADEON_RESET_CP))) {
3725 radeon_ring_lockup_update(ring);
3726 return false;
3727 }
3728 /* force CP activities */
3729 radeon_ring_force_activity(rdev, ring);
3730 return radeon_ring_test_lockup(rdev, ring);
3731}
3732
3733/**
3734 * si_dma_is_lockup - Check if the DMA engine is locked up
3735 *
3736 * @rdev: radeon_device pointer
3737 * @ring: radeon_ring structure holding ring information
3738 *
3739 * Check if the async DMA engine is locked up.
3740 * Returns true if the engine appears to be locked up, false if not.
3741 */
3742bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3743{
3744 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3745 u32 mask;
3746
3747 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3748 mask = RADEON_RESET_DMA;
3749 else
3750 mask = RADEON_RESET_DMA1;
3751
3752 if (!(reset_mask & mask)) {
3753 radeon_ring_lockup_update(ring);
3754 return false;
3755 }
3756 /* force ring activities */
3757 radeon_ring_force_activity(rdev, ring);
3758 return radeon_ring_test_lockup(rdev, ring);
3759}
3760
d2800ee5
AD		 3761/* MC */
3762static void si_mc_program(struct radeon_device *rdev)
3763{
3764 struct evergreen_mc_save save;
3765 u32 tmp;
3766 int i, j;
3767
3768 /* Initialize HDP */
3769 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3770 WREG32((0x2c14 + j), 0x00000000);
3771 WREG32((0x2c18 + j), 0x00000000);
3772 WREG32((0x2c1c + j), 0x00000000);
3773 WREG32((0x2c20 + j), 0x00000000);
3774 WREG32((0x2c24 + j), 0x00000000);
3775 }
3776 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3777
3778 evergreen_mc_stop(rdev, &save);
3779 if (radeon_mc_wait_for_idle(rdev)) {
3780 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3781 }
5153550a
AD		 3782 if (!ASIC_IS_NODCE(rdev))
3783 /* Lockout access through VGA aperture*/
3784 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3785 /* Update configuration */
3786 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3787 rdev->mc.vram_start >> 12);
3788 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3789 rdev->mc.vram_end >> 12);
3790 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3791 rdev->vram_scratch.gpu_addr >> 12);
3792 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3793 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3794 WREG32(MC_VM_FB_LOCATION, tmp);
3795 /* XXX double check these! */
3796 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3797 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3798 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3799 WREG32(MC_VM_AGP_BASE, 0);
3800 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3801 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3802 if (radeon_mc_wait_for_idle(rdev)) {
3803 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3804 }
3805 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3806 if (!ASIC_IS_NODCE(rdev)) {
3807 /* we need to own VRAM, so turn off the VGA renderer here
3808 * to stop it overwriting our objects */
3809 rv515_vga_render_disable(rdev);
3810 }
d2800ee5
AD		 3811}
3812
1c49165d
AD		 3813void si_vram_gtt_location(struct radeon_device *rdev,
3814 struct radeon_mc *mc)
d2800ee5
AD		 3815{
3816 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3817 /* leave room for at least 1024M GTT */
3818 dev_warn(rdev->dev, "limiting VRAM\n");
3819 mc->real_vram_size = 0xFFC0000000ULL;
3820 mc->mc_vram_size = 0xFFC0000000ULL;
3821 }
9ed8b1f9 3822 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3823 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3824 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3825}
3826
3827static int si_mc_init(struct radeon_device *rdev)
3828{
3829 u32 tmp;
3830 int chansize, numchan;
3831
3832 /* Get VRAM informations */
3833 rdev->mc.vram_is_ddr = true;
3834 tmp = RREG32(MC_ARB_RAMCFG);
3835 if (tmp & CHANSIZE_OVERRIDE) {
3836 chansize = 16;
3837 } else if (tmp & CHANSIZE_MASK) {
3838 chansize = 64;
3839 } else {
3840 chansize = 32;
3841 }
3842 tmp = RREG32(MC_SHARED_CHMAP);
3843 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3844 case 0:
3845 default:
3846 numchan = 1;
3847 break;
3848 case 1:
3849 numchan = 2;
3850 break;
3851 case 2:
3852 numchan = 4;
3853 break;
3854 case 3:
3855 numchan = 8;
3856 break;
3857 case 4:
3858 numchan = 3;
3859 break;
3860 case 5:
3861 numchan = 6;
3862 break;
3863 case 6:
3864 numchan = 10;
3865 break;
3866 case 7:
3867 numchan = 12;
3868 break;
3869 case 8:
3870 numchan = 16;
3871 break;
3872 }
3873 rdev->mc.vram_width = numchan * chansize;
3874 /* Could aper size report 0 ? */
3875 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3876 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3877 /* size in MB on si */
fc986034
NOS		 3878 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3879 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
d2800ee5
AD		 3880 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3881 si_vram_gtt_location(rdev, &rdev->mc);
3882 radeon_update_bandwidth_info(rdev);
3883
3884 return 0;
3885}
3886
3887/*
3888 * GART
3889 */
3890void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3891{
3892 /* flush hdp cache */
3893 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3894
3895 /* bits 0-15 are the VM contexts0-15 */
3896 WREG32(VM_INVALIDATE_REQUEST, 1);
3897}
3898
1109ca09 3899static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 3900{
3901 int r, i;
3902
3903 if (rdev->gart.robj == NULL) {
3904 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3905 return -EINVAL;
3906 }
3907 r = radeon_gart_table_vram_pin(rdev);
3908 if (r)
3909 return r;
3910 radeon_gart_restore(rdev);
3911 /* Setup TLB control */
3912 WREG32(MC_VM_MX_L1_TLB_CNTL,
3913 (0xA << 7) |
3914 ENABLE_L1_TLB |
3915 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3916 ENABLE_ADVANCED_DRIVER_MODEL |
3917 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3918 /* Setup L2 cache */
3919 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3920 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3921 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3922 EFFECTIVE_L2_QUEUE_SIZE(7) |
3923 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3924 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3925 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3926 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3927 /* setup context0 */
3928 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3929 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3930 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3931 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3932 (u32)(rdev->dummy_page.addr >> 12));
3933 WREG32(VM_CONTEXT0_CNTL2, 0);
3934 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3935 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3936
3937 WREG32(0x15D4, 0);
3938 WREG32(0x15D8, 0);
3939 WREG32(0x15DC, 0);
3940
3941 /* empty context1-15 */
d2800ee5
AD		 3942 /* set vm size, must be a multiple of 4 */
3943 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 3944 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 3945 /* Assign the pt base to something valid for now; the pts used for
3946 * the VMs are determined by the application and setup and assigned
3947 * on the fly in the vm part of radeon_gart.c
3948 */
d2800ee5
AD		 3949 for (i = 1; i < 16; i++) {
3950 if (i < 8)
3951 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3952 rdev->gart.table_addr >> 12);
3953 else
3954 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3955 rdev->gart.table_addr >> 12);
3956 }
3957
3958 /* enable context1-15 */
3959 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3960 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 3961 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 3962 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 3963 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3964 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3965 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3966 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3967 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3968 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3969 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3970 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3971 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3972 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3973 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3974 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 3975
3976 si_pcie_gart_tlb_flush(rdev);
3977 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3978 (unsigned)(rdev->mc.gtt_size >> 20),
3979 (unsigned long long)rdev->gart.table_addr);
3980 rdev->gart.ready = true;
3981 return 0;
3982}
3983
1109ca09 3984static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 3985{
3986 /* Disable all tables */
3987 WREG32(VM_CONTEXT0_CNTL, 0);
3988 WREG32(VM_CONTEXT1_CNTL, 0);
3989 /* Setup TLB control */
3990 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3991 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3992 /* Setup L2 cache */
3993 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3994 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3995 EFFECTIVE_L2_QUEUE_SIZE(7) |
3996 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3997 WREG32(VM_L2_CNTL2, 0);
3998 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3999 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4000 radeon_gart_table_vram_unpin(rdev);
4001}
4002
1109ca09 4003static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 4004{
4005 si_pcie_gart_disable(rdev);
4006 radeon_gart_table_vram_free(rdev);
4007 radeon_gart_fini(rdev);
4008}
4009
498dd8b3
AD		 4010/* vm parser */
4011static bool si_vm_reg_valid(u32 reg)
4012{
4013 /* context regs are fine */
4014 if (reg >= 0x28000)
4015 return true;
4016
4017 /* check config regs */
4018 switch (reg) {
4019 case GRBM_GFX_INDEX:
f418b88a 4020 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4021 case VGT_VTX_VECT_EJECT_REG:
4022 case VGT_CACHE_INVALIDATION:
4023 case VGT_ESGS_RING_SIZE:
4024 case VGT_GSVS_RING_SIZE:
4025 case VGT_GS_VERTEX_REUSE:
4026 case VGT_PRIMITIVE_TYPE:
4027 case VGT_INDEX_TYPE:
4028 case VGT_NUM_INDICES:
4029 case VGT_NUM_INSTANCES:
4030 case VGT_TF_RING_SIZE:
4031 case VGT_HS_OFFCHIP_PARAM:
4032 case VGT_TF_MEMORY_BASE:
4033 case PA_CL_ENHANCE:
4034 case PA_SU_LINE_STIPPLE_VALUE:
4035 case PA_SC_LINE_STIPPLE_STATE:
4036 case PA_SC_ENHANCE:
4037 case SQC_CACHES:
4038 case SPI_STATIC_THREAD_MGMT_1:
4039 case SPI_STATIC_THREAD_MGMT_2:
4040 case SPI_STATIC_THREAD_MGMT_3:
4041 case SPI_PS_MAX_WAVE_ID:
4042 case SPI_CONFIG_CNTL:
4043 case SPI_CONFIG_CNTL_1:
4044 case TA_CNTL_AUX:
4045 return true;
4046 default:
4047 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4048 return false;
4049 }
4050}
4051
4052static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4053 u32 *ib, struct radeon_cs_packet *pkt)
4054{
4055 switch (pkt->opcode) {
4056 case PACKET3_NOP:
4057 case PACKET3_SET_BASE:
4058 case PACKET3_SET_CE_DE_COUNTERS:
4059 case PACKET3_LOAD_CONST_RAM:
4060 case PACKET3_WRITE_CONST_RAM:
4061 case PACKET3_WRITE_CONST_RAM_OFFSET:
4062 case PACKET3_DUMP_CONST_RAM:
4063 case PACKET3_INCREMENT_CE_COUNTER:
4064 case PACKET3_WAIT_ON_DE_COUNTER:
4065 case PACKET3_CE_WRITE:
4066 break;
4067 default:
4068 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4069 return -EINVAL;
4070 }
4071 return 0;
4072}
4073
4074static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4075 u32 *ib, struct radeon_cs_packet *pkt)
4076{
4077 u32 idx = pkt->idx + 1;
4078 u32 idx_value = ib[idx];
4079 u32 start_reg, end_reg, reg, i;
5aa709be 4080 u32 command, info;
498dd8b3
AD		 4081
4082 switch (pkt->opcode) {
4083 case PACKET3_NOP:
4084 case PACKET3_SET_BASE:
4085 case PACKET3_CLEAR_STATE:
4086 case PACKET3_INDEX_BUFFER_SIZE:
4087 case PACKET3_DISPATCH_DIRECT:
4088 case PACKET3_DISPATCH_INDIRECT:
4089 case PACKET3_ALLOC_GDS:
4090 case PACKET3_WRITE_GDS_RAM:
4091 case PACKET3_ATOMIC_GDS:
4092 case PACKET3_ATOMIC:
4093 case PACKET3_OCCLUSION_QUERY:
4094 case PACKET3_SET_PREDICATION:
4095 case PACKET3_COND_EXEC:
4096 case PACKET3_PRED_EXEC:
4097 case PACKET3_DRAW_INDIRECT:
4098 case PACKET3_DRAW_INDEX_INDIRECT:
4099 case PACKET3_INDEX_BASE:
4100 case PACKET3_DRAW_INDEX_2:
4101 case PACKET3_CONTEXT_CONTROL:
4102 case PACKET3_INDEX_TYPE:
4103 case PACKET3_DRAW_INDIRECT_MULTI:
4104 case PACKET3_DRAW_INDEX_AUTO:
4105 case PACKET3_DRAW_INDEX_IMMD:
4106 case PACKET3_NUM_INSTANCES:
4107 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4108 case PACKET3_STRMOUT_BUFFER_UPDATE:
4109 case PACKET3_DRAW_INDEX_OFFSET_2:
4110 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4111 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4112 case PACKET3_MPEG_INDEX:
4113 case PACKET3_WAIT_REG_MEM:
4114 case PACKET3_MEM_WRITE:
4115 case PACKET3_PFP_SYNC_ME:
4116 case PACKET3_SURFACE_SYNC:
4117 case PACKET3_EVENT_WRITE:
4118 case PACKET3_EVENT_WRITE_EOP:
4119 case PACKET3_EVENT_WRITE_EOS:
4120 case PACKET3_SET_CONTEXT_REG:
4121 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4122 case PACKET3_SET_SH_REG:
4123 case PACKET3_SET_SH_REG_OFFSET:
4124 case PACKET3_INCREMENT_DE_COUNTER:
4125 case PACKET3_WAIT_ON_CE_COUNTER:
4126 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4127 case PACKET3_ME_WRITE:
4128 break;
4129 case PACKET3_COPY_DATA:
4130 if ((idx_value & 0xf00) == 0) {
4131 reg = ib[idx + 3] * 4;
4132 if (!si_vm_reg_valid(reg))
4133 return -EINVAL;
4134 }
4135 break;
4136 case PACKET3_WRITE_DATA:
4137 if ((idx_value & 0xf00) == 0) {
4138 start_reg = ib[idx + 1] * 4;
4139 if (idx_value & 0x10000) {
4140 if (!si_vm_reg_valid(start_reg))
4141 return -EINVAL;
4142 } else {
4143 for (i = 0; i < (pkt->count - 2); i++) {
4144 reg = start_reg + (4 * i);
4145 if (!si_vm_reg_valid(reg))
4146 return -EINVAL;
4147 }
4148 }
4149 }
4150 break;
4151 case PACKET3_COND_WRITE:
4152 if (idx_value & 0x100) {
4153 reg = ib[idx + 5] * 4;
4154 if (!si_vm_reg_valid(reg))
4155 return -EINVAL;
4156 }
4157 break;
4158 case PACKET3_COPY_DW:
4159 if (idx_value & 0x2) {
4160 reg = ib[idx + 3] * 4;
4161 if (!si_vm_reg_valid(reg))
4162 return -EINVAL;
4163 }
4164 break;
4165 case PACKET3_SET_CONFIG_REG:
4166 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4167 end_reg = 4 * pkt->count + start_reg - 4;
4168 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4169 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4170 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4171 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4172 return -EINVAL;
4173 }
4174 for (i = 0; i < pkt->count; i++) {
4175 reg = start_reg + (4 * i);
4176 if (!si_vm_reg_valid(reg))
4177 return -EINVAL;
4178 }
4179 break;
5aa709be
AD		 4180 case PACKET3_CP_DMA:
4181 command = ib[idx + 4];
4182 info = ib[idx + 1];
4183 if (command & PACKET3_CP_DMA_CMD_SAS) {
4184 /* src address space is register */
4185 if (((info & 0x60000000) >> 29) == 0) {
4186 start_reg = idx_value << 2;
4187 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4188 reg = start_reg;
4189 if (!si_vm_reg_valid(reg)) {
4190 DRM_ERROR("CP DMA Bad SRC register\n");
4191 return -EINVAL;
4192 }
4193 } else {
4194 for (i = 0; i < (command & 0x1fffff); i++) {
4195 reg = start_reg + (4 * i);
4196 if (!si_vm_reg_valid(reg)) {
4197 DRM_ERROR("CP DMA Bad SRC register\n");
4198 return -EINVAL;
4199 }
4200 }
4201 }
4202 }
4203 }
4204 if (command & PACKET3_CP_DMA_CMD_DAS) {
4205 /* dst address space is register */
4206 if (((info & 0x00300000) >> 20) == 0) {
4207 start_reg = ib[idx + 2];
4208 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4209 reg = start_reg;
4210 if (!si_vm_reg_valid(reg)) {
4211 DRM_ERROR("CP DMA Bad DST register\n");
4212 return -EINVAL;
4213 }
4214 } else {
4215 for (i = 0; i < (command & 0x1fffff); i++) {
4216 reg = start_reg + (4 * i);
4217 if (!si_vm_reg_valid(reg)) {
4218 DRM_ERROR("CP DMA Bad DST register\n");
4219 return -EINVAL;
4220 }
4221 }
4222 }
4223 }
4224 }
4225 break;
498dd8b3
AD		 4226 default:
4227 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4228 return -EINVAL;
4229 }
4230 return 0;
4231}
4232
4233static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4234 u32 *ib, struct radeon_cs_packet *pkt)
4235{
4236 u32 idx = pkt->idx + 1;
4237 u32 idx_value = ib[idx];
4238 u32 start_reg, reg, i;
4239
4240 switch (pkt->opcode) {
4241 case PACKET3_NOP:
4242 case PACKET3_SET_BASE:
4243 case PACKET3_CLEAR_STATE:
4244 case PACKET3_DISPATCH_DIRECT:
4245 case PACKET3_DISPATCH_INDIRECT:
4246 case PACKET3_ALLOC_GDS:
4247 case PACKET3_WRITE_GDS_RAM:
4248 case PACKET3_ATOMIC_GDS:
4249 case PACKET3_ATOMIC:
4250 case PACKET3_OCCLUSION_QUERY:
4251 case PACKET3_SET_PREDICATION:
4252 case PACKET3_COND_EXEC:
4253 case PACKET3_PRED_EXEC:
4254 case PACKET3_CONTEXT_CONTROL:
4255 case PACKET3_STRMOUT_BUFFER_UPDATE:
4256 case PACKET3_WAIT_REG_MEM:
4257 case PACKET3_MEM_WRITE:
4258 case PACKET3_PFP_SYNC_ME:
4259 case PACKET3_SURFACE_SYNC:
4260 case PACKET3_EVENT_WRITE:
4261 case PACKET3_EVENT_WRITE_EOP:
4262 case PACKET3_EVENT_WRITE_EOS:
4263 case PACKET3_SET_CONTEXT_REG:
4264 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4265 case PACKET3_SET_SH_REG:
4266 case PACKET3_SET_SH_REG_OFFSET:
4267 case PACKET3_INCREMENT_DE_COUNTER:
4268 case PACKET3_WAIT_ON_CE_COUNTER:
4269 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4270 case PACKET3_ME_WRITE:
4271 break;
4272 case PACKET3_COPY_DATA:
4273 if ((idx_value & 0xf00) == 0) {
4274 reg = ib[idx + 3] * 4;
4275 if (!si_vm_reg_valid(reg))
4276 return -EINVAL;
4277 }
4278 break;
4279 case PACKET3_WRITE_DATA:
4280 if ((idx_value & 0xf00) == 0) {
4281 start_reg = ib[idx + 1] * 4;
4282 if (idx_value & 0x10000) {
4283 if (!si_vm_reg_valid(start_reg))
4284 return -EINVAL;
4285 } else {
4286 for (i = 0; i < (pkt->count - 2); i++) {
4287 reg = start_reg + (4 * i);
4288 if (!si_vm_reg_valid(reg))
4289 return -EINVAL;
4290 }
4291 }
4292 }
4293 break;
4294 case PACKET3_COND_WRITE:
4295 if (idx_value & 0x100) {
4296 reg = ib[idx + 5] * 4;
4297 if (!si_vm_reg_valid(reg))
4298 return -EINVAL;
4299 }
4300 break;
4301 case PACKET3_COPY_DW:
4302 if (idx_value & 0x2) {
4303 reg = ib[idx + 3] * 4;
4304 if (!si_vm_reg_valid(reg))
4305 return -EINVAL;
4306 }
4307 break;
4308 default:
4309 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4310 return -EINVAL;
4311 }
4312 return 0;
4313}
4314
4315int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4316{
4317 int ret = 0;
4318 u32 idx = 0;
4319 struct radeon_cs_packet pkt;
4320
4321 do {
4322 pkt.idx = idx;
4e872ae2
IH		 4323 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4324 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4325 pkt.one_reg_wr = 0;
4326 switch (pkt.type) {
4e872ae2 4327 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4328 dev_err(rdev->dev, "Packet0 not allowed!\n");
4329 ret = -EINVAL;
4330 break;
4e872ae2 4331 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4332 idx += 1;
4333 break;
4e872ae2
IH		 4334 case RADEON_PACKET_TYPE3:
4335 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4336 if (ib->is_const_ib)
4337 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4338 else {
876dc9f3 4339 switch (ib->ring) {
498dd8b3
AD		 4340 case RADEON_RING_TYPE_GFX_INDEX:
4341 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4342 break;
4343 case CAYMAN_RING_TYPE_CP1_INDEX:
4344 case CAYMAN_RING_TYPE_CP2_INDEX:
4345 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4346 break;
4347 default:
876dc9f3 4348 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4349 ret = -EINVAL;
4350 break;
4351 }
4352 }
4353 idx += pkt.count + 2;
4354 break;
4355 default:
4356 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4357 ret = -EINVAL;
4358 break;
4359 }
4360 if (ret)
4361 break;
4362 } while (idx < ib->length_dw);
4363
4364 return ret;
4365}
4366
d2800ee5
AD		 4367/*
4368 * vm
4369 */
4370int si_vm_init(struct radeon_device *rdev)
4371{
4372 /* number of VMs */
4373 rdev->vm_manager.nvm = 16;
4374 /* base offset of vram pages */
4375 rdev->vm_manager.vram_base_offset = 0;
4376
4377 return 0;
4378}
4379
4380void si_vm_fini(struct radeon_device *rdev)
4381{
4382}
4383
82ffd92b
AD		 4384/**
4385 * si_vm_set_page - update the page tables using the CP
4386 *
4387 * @rdev: radeon_device pointer
43f1214a 4388 * @ib: indirect buffer to fill with commands
82ffd92b
AD		 4389 * @pe: addr of the page entry
4390 * @addr: dst addr to write into pe
4391 * @count: number of page entries to update
4392 * @incr: increase next addr by incr bytes
4393 * @flags: access flags
4394 *
43f1214a 4395 * Update the page tables using the CP (SI).
82ffd92b 4396 */
43f1214a
AD		 4397void si_vm_set_page(struct radeon_device *rdev,
4398 struct radeon_ib *ib,
4399 uint64_t pe,
82ffd92b
AD		 4400 uint64_t addr, unsigned count,
4401 uint32_t incr, uint32_t flags)
d2800ee5 4402{
82ffd92b 4403 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
deab48f1
AD		 4404 uint64_t value;
4405 unsigned ndw;
4406
4407 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4408 while (count) {
4409 ndw = 2 + count * 2;
4410 if (ndw > 0x3FFE)
4411 ndw = 0x3FFE;
4412
43f1214a
AD		 4413 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4414 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4415 WRITE_DATA_DST_SEL(1));
4416 ib->ptr[ib->length_dw++] = pe;
4417 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
deab48f1
AD		 4418 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4419 if (flags & RADEON_VM_PAGE_SYSTEM) {
4420 value = radeon_vm_map_gart(rdev, addr);
4421 value &= 0xFFFFFFFFFFFFF000ULL;
4422 } else if (flags & RADEON_VM_PAGE_VALID) {
4423 value = addr;
4424 } else {
4425 value = 0;
4426 }
4427 addr += incr;
4428 value |= r600_flags;
43f1214a
AD		 4429 ib->ptr[ib->length_dw++] = value;
4430 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4431 }
4432 }
4433 } else {
4434 /* DMA */
4435 if (flags & RADEON_VM_PAGE_SYSTEM) {
4436 while (count) {
4437 ndw = count * 2;
4438 if (ndw > 0xFFFFE)
4439 ndw = 0xFFFFE;
4440
4441 /* for non-physically contiguous pages (system) */
43f1214a
AD		 4442 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
4443 ib->ptr[ib->length_dw++] = pe;
4444 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
deab48f1
AD		 4445 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
4446 if (flags & RADEON_VM_PAGE_SYSTEM) {
4447 value = radeon_vm_map_gart(rdev, addr);
4448 value &= 0xFFFFFFFFFFFFF000ULL;
4449 } else if (flags & RADEON_VM_PAGE_VALID) {
4450 value = addr;
4451 } else {
4452 value = 0;
4453 }
4454 addr += incr;
4455 value |= r600_flags;
43f1214a
AD		 4456 ib->ptr[ib->length_dw++] = value;
4457 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4458 }
4459 }
4460 } else {
4461 while (count) {
4462 ndw = count * 2;
4463 if (ndw > 0xFFFFE)
4464 ndw = 0xFFFFE;
4465
4466 if (flags & RADEON_VM_PAGE_VALID)
4467 value = addr;
4468 else
4469 value = 0;
4470 /* for physically contiguous pages (vram) */
43f1214a
AD		 4471 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
4472 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4473 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4474 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4475 ib->ptr[ib->length_dw++] = 0;
4476 ib->ptr[ib->length_dw++] = value; /* value */
4477 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4478 ib->ptr[ib->length_dw++] = incr; /* increment size */
4479 ib->ptr[ib->length_dw++] = 0;
deab48f1
AD		 4480 pe += ndw * 4;
4481 addr += (ndw / 2) * incr;
4482 count -= ndw / 2;
4483 }
d7025d89 4484 }
43f1214a
AD		 4485 while (ib->length_dw & 0x7)
4486 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
82ffd92b 4487 }
d2800ee5
AD		 4488}
4489
498522b4 4490void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4491{
498522b4 4492 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4493
ee60e29f 4494 if (vm == NULL)
d2800ee5
AD		 4495 return;
4496
76c44f2c
AD		 4497 /* write new base address */
4498 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4499 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4500 WRITE_DATA_DST_SEL(0)));
4501
ee60e29f 4502 if (vm->id < 8) {
76c44f2c
AD		 4503 radeon_ring_write(ring,
4504 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4505 } else {
76c44f2c
AD		 4506 radeon_ring_write(ring,
4507 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4508 }
76c44f2c 4509 radeon_ring_write(ring, 0);
fa87e62d 4510 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4511
d2800ee5 4512 /* flush hdp cache */
76c44f2c
AD		 4513 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4514 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4515 WRITE_DATA_DST_SEL(0)));
4516 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4517 radeon_ring_write(ring, 0);
ee60e29f
CK		 4518 radeon_ring_write(ring, 0x1);
4519
d2800ee5 4520 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4521 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4522 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4523 WRITE_DATA_DST_SEL(0)));
4524 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4525 radeon_ring_write(ring, 0);
498522b4 4526 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4527
4528 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4529 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4530 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4531}
4532
8c5fd7ef
AD		 4533void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4534{
4535 struct radeon_ring *ring = &rdev->ring[ridx];
4536
4537 if (vm == NULL)
4538 return;
4539
4540 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4541 if (vm->id < 8) {
4542 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4543 } else {
4544 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4545 }
4546 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4547
4548 /* flush hdp cache */
4549 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4550 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4551 radeon_ring_write(ring, 1);
4552
4553 /* bits 0-7 are the VM contexts0-7 */
4554 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4555 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4556 radeon_ring_write(ring, 1 << vm->id);
4557}
4558
f8f84ac5
AD		 4559/*
4560 * Power and clock gating
4561 */
4562static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4563{
4564 int i;
4565
4566 for (i = 0; i < rdev->usec_timeout; i++) {
4567 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4568 break;
4569 udelay(1);
4570 }
4571
4572 for (i = 0; i < rdev->usec_timeout; i++) {
4573 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4574 break;
4575 udelay(1);
4576 }
4577}
4578
4579static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4580 bool enable)
4581{
4582 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4583 u32 mask;
4584 int i;
4585
4586 if (enable)
4587 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4588 else
4589 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4590 WREG32(CP_INT_CNTL_RING0, tmp);
4591
4592 if (!enable) {
4593 /* read a gfx register */
4594 tmp = RREG32(DB_DEPTH_INFO);
4595
4596 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4597 for (i = 0; i < rdev->usec_timeout; i++) {
4598 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4599 break;
4600 udelay(1);
4601 }
4602 }
4603}
4604
4605static void si_set_uvd_dcm(struct radeon_device *rdev,
4606 bool sw_mode)
4607{
4608 u32 tmp, tmp2;
4609
4610 tmp = RREG32(UVD_CGC_CTRL);
4611 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4612 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4613
4614 if (sw_mode) {
4615 tmp &= ~0x7ffff800;
4616 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4617 } else {
4618 tmp |= 0x7ffff800;
4619 tmp2 = 0;
4620 }
4621
4622 WREG32(UVD_CGC_CTRL, tmp);
4623 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4624}
4625
4626static void si_init_uvd_internal_cg(struct radeon_device *rdev)
4627{
4628 bool hw_mode = true;
4629
4630 if (hw_mode) {
4631 si_set_uvd_dcm(rdev, false);
4632 } else {
4633 u32 tmp = RREG32(UVD_CGC_CTRL);
4634 tmp &= ~DCM;
4635 WREG32(UVD_CGC_CTRL, tmp);
4636 }
4637}
4638
4639static u32 si_halt_rlc(struct radeon_device *rdev)
4640{
4641 u32 data, orig;
4642
4643 orig = data = RREG32(RLC_CNTL);
4644
4645 if (data & RLC_ENABLE) {
4646 data &= ~RLC_ENABLE;
4647 WREG32(RLC_CNTL, data);
4648
4649 si_wait_for_rlc_serdes(rdev);
4650 }
4651
4652 return orig;
4653}
4654
4655static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4656{
4657 u32 tmp;
4658
4659 tmp = RREG32(RLC_CNTL);
4660 if (tmp != rlc)
4661 WREG32(RLC_CNTL, rlc);
4662}
4663
4664static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4665{
4666 u32 data, orig;
4667
4668 orig = data = RREG32(DMA_PG);
4669 if (enable)
4670 data |= PG_CNTL_ENABLE;
4671 else
4672 data &= ~PG_CNTL_ENABLE;
4673 if (orig != data)
4674 WREG32(DMA_PG, data);
4675}
4676
4677static void si_init_dma_pg(struct radeon_device *rdev)
4678{
4679 u32 tmp;
4680
4681 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4682 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4683
4684 for (tmp = 0; tmp < 5; tmp++)
4685 WREG32(DMA_PGFSM_WRITE, 0);
4686}
4687
4688static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4689 bool enable)
4690{
4691 u32 tmp;
4692
4693 if (enable) {
4694 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4695 WREG32(RLC_TTOP_D, tmp);
4696
4697 tmp = RREG32(RLC_PG_CNTL);
4698 tmp |= GFX_PG_ENABLE;
4699 WREG32(RLC_PG_CNTL, tmp);
4700
4701 tmp = RREG32(RLC_AUTO_PG_CTRL);
4702 tmp |= AUTO_PG_EN;
4703 WREG32(RLC_AUTO_PG_CTRL, tmp);
4704 } else {
4705 tmp = RREG32(RLC_AUTO_PG_CTRL);
4706 tmp &= ~AUTO_PG_EN;
4707 WREG32(RLC_AUTO_PG_CTRL, tmp);
4708
4709 tmp = RREG32(DB_RENDER_CONTROL);
4710 }
4711}
4712
4713static void si_init_gfx_cgpg(struct radeon_device *rdev)
4714{
4715 u32 tmp;
4716
4717 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4718
4719 tmp = RREG32(RLC_PG_CNTL);
4720 tmp |= GFX_PG_SRC;
4721 WREG32(RLC_PG_CNTL, tmp);
4722
4723 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4724
4725 tmp = RREG32(RLC_AUTO_PG_CTRL);
4726
4727 tmp &= ~GRBM_REG_SGIT_MASK;
4728 tmp |= GRBM_REG_SGIT(0x700);
4729 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4730 WREG32(RLC_AUTO_PG_CTRL, tmp);
4731}
4732
4733static u32 get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
4734{
4735 u32 mask = 0, tmp, tmp1;
4736 int i;
4737
4738 si_select_se_sh(rdev, se, sh);
4739 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4740 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4741 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4742
4743 tmp &= 0xffff0000;
4744
4745 tmp |= tmp1;
4746 tmp >>= 16;
4747
4748 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4749 mask <<= 1;
4750 mask |= 1;
4751 }
4752
4753 return (~tmp) & mask;
4754}
4755
4756static void si_init_ao_cu_mask(struct radeon_device *rdev)
4757{
4758 u32 i, j, k, active_cu_number = 0;
4759 u32 mask, counter, cu_bitmap;
4760 u32 tmp = 0;
4761
4762 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4763 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4764 mask = 1;
4765 cu_bitmap = 0;
4766 counter = 0;
4767 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
4768 if (get_cu_active_bitmap(rdev, i, j) & mask) {
4769 if (counter < 2)
4770 cu_bitmap |= mask;
4771 counter++;
4772 }
4773 mask <<= 1;
4774 }
4775
4776 active_cu_number += counter;
4777 tmp |= (cu_bitmap << (i * 16 + j * 8));
4778 }
4779 }
4780
4781 WREG32(RLC_PG_AO_CU_MASK, tmp);
4782
4783 tmp = RREG32(RLC_MAX_PG_CU);
4784 tmp &= ~MAX_PU_CU_MASK;
4785 tmp |= MAX_PU_CU(active_cu_number);
4786 WREG32(RLC_MAX_PG_CU, tmp);
4787}
4788
4789static void si_enable_cgcg(struct radeon_device *rdev,
4790 bool enable)
4791{
4792 u32 data, orig, tmp;
4793
4794 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
4795
4796 si_enable_gui_idle_interrupt(rdev, enable);
4797
4798 if (enable) {
4799 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
4800
4801 tmp = si_halt_rlc(rdev);
4802
4803 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4804 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4805 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
4806
4807 si_wait_for_rlc_serdes(rdev);
4808
4809 si_update_rlc(rdev, tmp);
4810
4811 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
4812
4813 data |= CGCG_EN | CGLS_EN;
4814 } else {
4815 RREG32(CB_CGTT_SCLK_CTRL);
4816 RREG32(CB_CGTT_SCLK_CTRL);
4817 RREG32(CB_CGTT_SCLK_CTRL);
4818 RREG32(CB_CGTT_SCLK_CTRL);
4819
4820 data &= ~(CGCG_EN | CGLS_EN);
4821 }
4822
4823 if (orig != data)
4824 WREG32(RLC_CGCG_CGLS_CTRL, data);
4825}
4826
4827static void si_enable_mgcg(struct radeon_device *rdev,
4828 bool enable)
4829{
4830 u32 data, orig, tmp = 0;
4831
4832 if (enable) {
4833 orig = data = RREG32(CGTS_SM_CTRL_REG);
4834 data = 0x96940200;
4835 if (orig != data)
4836 WREG32(CGTS_SM_CTRL_REG, data);
4837
4838 orig = data = RREG32(CP_MEM_SLP_CNTL);
4839 data |= CP_MEM_LS_EN;
4840 if (orig != data)
4841 WREG32(CP_MEM_SLP_CNTL, data);
4842
4843 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4844 data &= 0xffffffc0;
4845 if (orig != data)
4846 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
4847
4848 tmp = si_halt_rlc(rdev);
4849
4850 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4851 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4852 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
4853
4854 si_update_rlc(rdev, tmp);
4855 } else {
4856 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4857 data |= 0x00000003;
4858 if (orig != data)
4859 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
4860
4861 data = RREG32(CP_MEM_SLP_CNTL);
4862 if (data & CP_MEM_LS_EN) {
4863 data &= ~CP_MEM_LS_EN;
4864 WREG32(CP_MEM_SLP_CNTL, data);
4865 }
4866 orig = data = RREG32(CGTS_SM_CTRL_REG);
4867 data |= LS_OVERRIDE | OVERRIDE;
4868 if (orig != data)
4869 WREG32(CGTS_SM_CTRL_REG, data);
4870
4871 tmp = si_halt_rlc(rdev);
4872
4873 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4874 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4875 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
4876
4877 si_update_rlc(rdev, tmp);
4878 }
4879}
4880
4881static void si_enable_uvd_mgcg(struct radeon_device *rdev,
4882 bool enable)
4883{
4884 u32 orig, data, tmp;
4885
4886 if (enable) {
4887 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
4888 tmp |= 0x3fff;
4889 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
4890
4891 orig = data = RREG32(UVD_CGC_CTRL);
4892 data |= DCM;
4893 if (orig != data)
4894 WREG32(UVD_CGC_CTRL, data);
4895
4896 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
4897 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
4898 } else {
4899 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
4900 tmp &= ~0x3fff;
4901 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
4902
4903 orig = data = RREG32(UVD_CGC_CTRL);
4904 data &= ~DCM;
4905 if (orig != data)
4906 WREG32(UVD_CGC_CTRL, data);
4907
4908 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
4909 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
4910 }
4911}
4912
4913static const u32 mc_cg_registers[] =
4914{
4915 MC_HUB_MISC_HUB_CG,
4916 MC_HUB_MISC_SIP_CG,
4917 MC_HUB_MISC_VM_CG,
4918 MC_XPB_CLK_GAT,
4919 ATC_MISC_CG,
4920 MC_CITF_MISC_WR_CG,
4921 MC_CITF_MISC_RD_CG,
4922 MC_CITF_MISC_VM_CG,
4923 VM_L2_CG,
4924};
4925
4926static void si_enable_mc_ls(struct radeon_device *rdev,
4927 bool enable)
4928{
4929 int i;
4930 u32 orig, data;
4931
4932 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
4933 orig = data = RREG32(mc_cg_registers[i]);
4934 if (enable)
4935 data |= MC_LS_ENABLE;
4936 else
4937 data &= ~MC_LS_ENABLE;
4938 if (data != orig)
4939 WREG32(mc_cg_registers[i], data);
4940 }
4941}
4942
4943
4944static void si_init_cg(struct radeon_device *rdev)
4945{
4946 bool has_uvd = true;
4947
4948 si_enable_mgcg(rdev, true);
4949 si_enable_cgcg(rdev, true);
4950 /* disable MC LS on Tahiti */
4951 if (rdev->family == CHIP_TAHITI)
4952 si_enable_mc_ls(rdev, false);
4953 if (has_uvd) {
4954 si_enable_uvd_mgcg(rdev, true);
4955 si_init_uvd_internal_cg(rdev);
4956 }
4957}
4958
4959static void si_fini_cg(struct radeon_device *rdev)
4960{
4961 bool has_uvd = true;
4962
4963 if (has_uvd)
4964 si_enable_uvd_mgcg(rdev, false);
4965 si_enable_cgcg(rdev, false);
4966 si_enable_mgcg(rdev, false);
4967}
4968
4969static void si_init_pg(struct radeon_device *rdev)
4970{
4971 bool has_pg = false;
4972
4973 /* only cape verde supports PG */
4974 if (rdev->family == CHIP_VERDE)
4975 has_pg = true;
4976
4977 if (has_pg) {
4978 si_init_ao_cu_mask(rdev);
4979 si_init_dma_pg(rdev);
4980 si_enable_dma_pg(rdev, true);
4981 si_init_gfx_cgpg(rdev);
4982 si_enable_gfx_cgpg(rdev, true);
4983 } else {
4984 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4985 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4986 }
4987}
4988
4989static void si_fini_pg(struct radeon_device *rdev)
4990{
4991 bool has_pg = false;
4992
4993 /* only cape verde supports PG */
4994 if (rdev->family == CHIP_VERDE)
4995 has_pg = true;
4996
4997 if (has_pg) {
4998 si_enable_dma_pg(rdev, false);
4999 si_enable_gfx_cgpg(rdev, false);
5000 }
5001}
5002
347e7592
AD		 5003/*
5004 * RLC
5005 */
c420c745 5006void si_rlc_fini(struct radeon_device *rdev)
347e7592
AD		 5007{
5008 int r;
5009
5010 /* save restore block */
5011 if (rdev->rlc.save_restore_obj) {
5012 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
5013 if (unlikely(r != 0))
5014 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
5015 radeon_bo_unpin(rdev->rlc.save_restore_obj);
5016 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
5017
5018 radeon_bo_unref(&rdev->rlc.save_restore_obj);
5019 rdev->rlc.save_restore_obj = NULL;
5020 }
5021
5022 /* clear state block */
5023 if (rdev->rlc.clear_state_obj) {
5024 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
5025 if (unlikely(r != 0))
5026 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
5027 radeon_bo_unpin(rdev->rlc.clear_state_obj);
5028 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
5029
5030 radeon_bo_unref(&rdev->rlc.clear_state_obj);
5031 rdev->rlc.clear_state_obj = NULL;
5032 }
5033}
5034
bd8cd539
AD		 5035#define RLC_CLEAR_STATE_END_MARKER 0x00000001
5036
c420c745 5037int si_rlc_init(struct radeon_device *rdev)
347e7592 5038{
6d8cf000 5039 volatile u32 *dst_ptr;
bd8cd539
AD		 5040 u32 dws, data, i, j, k, reg_num;
5041 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
5042 u64 reg_list_mc_addr;
5043 const struct cs_section_def *cs_data = si_cs_data;
5044 int r;
347e7592
AD		 5045
5046 /* save restore block */
5047 if (rdev->rlc.save_restore_obj == NULL) {
5048 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
40f5cf99
AD		 5049 RADEON_GEM_DOMAIN_VRAM, NULL,
5050 &rdev->rlc.save_restore_obj);
347e7592
AD		 5051 if (r) {
5052 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
5053 return r;
5054 }
5055 }
5056
5057 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
5058 if (unlikely(r != 0)) {
5059 si_rlc_fini(rdev);
5060 return r;
5061 }
5062 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
5063 &rdev->rlc.save_restore_gpu_addr);
5064 if (r) {
6d8cf000 5065 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
347e7592
AD		 5066 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
5067 si_rlc_fini(rdev);
5068 return r;
5069 }
5070
6d8cf000
AD		 5071 if (rdev->family == CHIP_VERDE) {
5072 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
5073 if (r) {
5074 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
5075 si_rlc_fini(rdev);
5076 return r;
5077 }
5078 /* write the sr buffer */
5079 dst_ptr = rdev->rlc.sr_ptr;
5080 for (i = 0; i < ARRAY_SIZE(verde_rlc_save_restore_register_list); i++) {
5081 dst_ptr[i] = verde_rlc_save_restore_register_list[i];
5082 }
5083 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
5084 }
5085 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
5086
347e7592 5087 /* clear state block */
bd8cd539
AD		 5088 reg_list_num = 0;
5089 dws = 0;
5090 for (i = 0; cs_data[i].section != NULL; i++) {
5091 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
5092 reg_list_num++;
5093 dws += cs_data[i].section[j].reg_count;
5094 }
5095 }
5096 reg_list_blk_index = (3 * reg_list_num + 2);
5097 dws += reg_list_blk_index;
5098
347e7592 5099 if (rdev->rlc.clear_state_obj == NULL) {
bd8cd539
AD		 5100 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
5101 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
347e7592
AD		 5102 if (r) {
5103 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
5104 si_rlc_fini(rdev);
5105 return r;
5106 }
5107 }
5108 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
5109 if (unlikely(r != 0)) {
5110 si_rlc_fini(rdev);
5111 return r;
5112 }
5113 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
5114 &rdev->rlc.clear_state_gpu_addr);
5115 if (r) {
bd8cd539
AD		 5116
5117 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
347e7592
AD		 5118 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
5119 si_rlc_fini(rdev);
5120 return r;
5121 }
bd8cd539
AD		 5122 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
5123 if (r) {
5124 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
5125 si_rlc_fini(rdev);
5126 return r;
5127 }
5128 /* set up the cs buffer */
5129 dst_ptr = rdev->rlc.cs_ptr;
5130 reg_list_hdr_blk_index = 0;
5131 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
5132 data = upper_32_bits(reg_list_mc_addr);
5133 dst_ptr[reg_list_hdr_blk_index] = data;
5134 reg_list_hdr_blk_index++;
5135 for (i = 0; cs_data[i].section != NULL; i++) {
5136 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
5137 reg_num = cs_data[i].section[j].reg_count;
5138 data = reg_list_mc_addr & 0xffffffff;
5139 dst_ptr[reg_list_hdr_blk_index] = data;
5140 reg_list_hdr_blk_index++;
5141
5142 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
5143 dst_ptr[reg_list_hdr_blk_index] = data;
5144 reg_list_hdr_blk_index++;
5145
5146 data = 0x08000000 | (reg_num * 4);
5147 dst_ptr[reg_list_hdr_blk_index] = data;
5148 reg_list_hdr_blk_index++;
5149
5150 for (k = 0; k < reg_num; k++) {
5151 data = cs_data[i].section[j].extent[k];
5152 dst_ptr[reg_list_blk_index + k] = data;
5153 }
5154 reg_list_mc_addr += reg_num * 4;
5155 reg_list_blk_index += reg_num;
5156 }
5157 }
5158 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
5159
5160 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
5161 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
347e7592
AD		 5162
5163 return 0;
5164}
5165
f8f84ac5 5166static void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5167{
f8f84ac5 5168 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5169
f8f84ac5
AD		 5170 tmp |= SOFT_RESET_RLC;
5171 WREG32(GRBM_SOFT_RESET, tmp);
5172 udelay(50);
5173 tmp &= ~SOFT_RESET_RLC;
5174 WREG32(GRBM_SOFT_RESET, tmp);
5175 udelay(50);
d719cef3
AD		 5176}
5177
347e7592
AD		 5178static void si_rlc_stop(struct radeon_device *rdev)
5179{
5180 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5181
5182 si_enable_gui_idle_interrupt(rdev, false);
5183
5184 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5185}
5186
5187static void si_rlc_start(struct radeon_device *rdev)
5188{
5189 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5190
5191 si_enable_gui_idle_interrupt(rdev, true);
5192
5193 udelay(50);
5194}
5195
5196static bool si_lbpw_supported(struct radeon_device *rdev)
5197{
5198 u32 tmp;
5199
5200 /* Enable LBPW only for DDR3 */
5201 tmp = RREG32(MC_SEQ_MISC0);
5202 if ((tmp & 0xF0000000) == 0xB0000000)
5203 return true;
5204 return false;
5205}
5206
5207static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5208{
5209 u32 tmp;
5210
5211 tmp = RREG32(RLC_LB_CNTL);
5212 if (enable)
5213 tmp |= LOAD_BALANCE_ENABLE;
5214 else
5215 tmp &= ~LOAD_BALANCE_ENABLE;
5216 WREG32(RLC_LB_CNTL, tmp);
5217
5218 if (!enable) {
5219 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5220 WREG32(SPI_LB_CU_MASK, 0x00ff);
5221 }
347e7592
AD		 5222}
5223
5224static int si_rlc_resume(struct radeon_device *rdev)
5225{
5226 u32 i;
5227 const __be32 *fw_data;
5228
5229 if (!rdev->rlc_fw)
5230 return -EINVAL;
5231
5232 si_rlc_stop(rdev);
5233
f8f84ac5
AD		 5234 si_rlc_reset(rdev);
5235
5236 si_init_pg(rdev);
5237
5238 si_init_cg(rdev);
5239
347e7592
AD		 5240 WREG32(RLC_RL_BASE, 0);
5241 WREG32(RLC_RL_SIZE, 0);
5242 WREG32(RLC_LB_CNTL, 0);
5243 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5244 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5245 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5246
347e7592
AD		 5247 WREG32(RLC_MC_CNTL, 0);
5248 WREG32(RLC_UCODE_CNTL, 0);
5249
5250 fw_data = (const __be32 *)rdev->rlc_fw->data;
5251 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5252 WREG32(RLC_UCODE_ADDR, i);
5253 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5254 }
5255 WREG32(RLC_UCODE_ADDR, 0);
5256
d719cef3
AD		 5257 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5258
347e7592
AD		 5259 si_rlc_start(rdev);
5260
5261 return 0;
5262}
5263
25a857fb
AD		 5264static void si_enable_interrupts(struct radeon_device *rdev)
5265{
5266 u32 ih_cntl = RREG32(IH_CNTL);
5267 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5268
5269 ih_cntl |= ENABLE_INTR;
5270 ih_rb_cntl |= IH_RB_ENABLE;
5271 WREG32(IH_CNTL, ih_cntl);
5272 WREG32(IH_RB_CNTL, ih_rb_cntl);
5273 rdev->ih.enabled = true;
5274}
5275
5276static void si_disable_interrupts(struct radeon_device *rdev)
5277{
5278 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5279 u32 ih_cntl = RREG32(IH_CNTL);
5280
5281 ih_rb_cntl &= ~IH_RB_ENABLE;
5282 ih_cntl &= ~ENABLE_INTR;
5283 WREG32(IH_RB_CNTL, ih_rb_cntl);
5284 WREG32(IH_CNTL, ih_cntl);
5285 /* set rptr, wptr to 0 */
5286 WREG32(IH_RB_RPTR, 0);
5287 WREG32(IH_RB_WPTR, 0);
5288 rdev->ih.enabled = false;
25a857fb
AD		 5289 rdev->ih.rptr = 0;
5290}
5291
5292static void si_disable_interrupt_state(struct radeon_device *rdev)
5293{
5294 u32 tmp;
5295
5296 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5297 WREG32(CP_INT_CNTL_RING1, 0);
5298 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5299 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5300 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5301 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5302 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5303 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 5304 if (rdev->num_crtc >= 2) {
5305 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5306 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5307 }
25a857fb
AD		 5308 if (rdev->num_crtc >= 4) {
5309 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5310 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5311 }
5312 if (rdev->num_crtc >= 6) {
5313 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5314 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5315 }
5316
5153550a
AD		 5317 if (rdev->num_crtc >= 2) {
5318 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5319 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5320 }
25a857fb
AD		 5321 if (rdev->num_crtc >= 4) {
5322 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5323 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5324 }
5325 if (rdev->num_crtc >= 6) {
5326 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5327 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5328 }
5329
5153550a
AD		 5330 if (!ASIC_IS_NODCE(rdev)) {
5331 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5332
5333 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5334 WREG32(DC_HPD1_INT_CONTROL, tmp);
5335 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5336 WREG32(DC_HPD2_INT_CONTROL, tmp);
5337 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5338 WREG32(DC_HPD3_INT_CONTROL, tmp);
5339 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5340 WREG32(DC_HPD4_INT_CONTROL, tmp);
5341 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5342 WREG32(DC_HPD5_INT_CONTROL, tmp);
5343 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5344 WREG32(DC_HPD6_INT_CONTROL, tmp);
5345 }
25a857fb
AD		 5346}
5347
5348static int si_irq_init(struct radeon_device *rdev)
5349{
5350 int ret = 0;
5351 int rb_bufsz;
5352 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5353
5354 /* allocate ring */
5355 ret = r600_ih_ring_alloc(rdev);
5356 if (ret)
5357 return ret;
5358
5359 /* disable irqs */
5360 si_disable_interrupts(rdev);
5361
5362 /* init rlc */
5363 ret = si_rlc_resume(rdev);
5364 if (ret) {
5365 r600_ih_ring_fini(rdev);
5366 return ret;
5367 }
5368
5369 /* setup interrupt control */
5370 /* set dummy read address to ring address */
5371 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5372 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5373 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5374 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5375 */
5376 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5377 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5378 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5379 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5380
5381 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5382 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
5383
5384 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5385 IH_WPTR_OVERFLOW_CLEAR |
5386 (rb_bufsz << 1));
5387
5388 if (rdev->wb.enabled)
5389 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5390
5391 /* set the writeback address whether it's enabled or not */
5392 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5393 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5394
5395 WREG32(IH_RB_CNTL, ih_rb_cntl);
5396
5397 /* set rptr, wptr to 0 */
5398 WREG32(IH_RB_RPTR, 0);
5399 WREG32(IH_RB_WPTR, 0);
5400
5401 /* Default settings for IH_CNTL (disabled at first) */
5402 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5403 /* RPTR_REARM only works if msi's are enabled */
5404 if (rdev->msi_enabled)
5405 ih_cntl |= RPTR_REARM;
5406 WREG32(IH_CNTL, ih_cntl);
5407
5408 /* force the active interrupt state to all disabled */
5409 si_disable_interrupt_state(rdev);
5410
2099810f
DA		 5411 pci_set_master(rdev->pdev);
5412
25a857fb
AD		 5413 /* enable irqs */
5414 si_enable_interrupts(rdev);
5415
5416 return ret;
5417}
5418
5419int si_irq_set(struct radeon_device *rdev)
5420{
5421 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
5422 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5423 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 5424 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 5425 u32 grbm_int_cntl = 0;
5426 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 5427 u32 dma_cntl, dma_cntl1;
25a857fb
AD		 5428
5429 if (!rdev->irq.installed) {
5430 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5431 return -EINVAL;
5432 }
5433 /* don't enable anything if the ih is disabled */
5434 if (!rdev->ih.enabled) {
5435 si_disable_interrupts(rdev);
5436 /* force the active interrupt state to all disabled */
5437 si_disable_interrupt_state(rdev);
5438 return 0;
5439 }
5440
5153550a
AD		 5441 if (!ASIC_IS_NODCE(rdev)) {
5442 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5443 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5444 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5445 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5446 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5447 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5448 }
25a857fb 5449
8c5fd7ef
AD		 5450 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5451 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5452
25a857fb 5453 /* enable CP interrupts on all rings */
736fc37f 5454 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 5455 DRM_DEBUG("si_irq_set: sw int gfx\n");
5456 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5457 }
736fc37f 5458 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 5459 DRM_DEBUG("si_irq_set: sw int cp1\n");
5460 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5461 }
736fc37f 5462 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 5463 DRM_DEBUG("si_irq_set: sw int cp2\n");
5464 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5465 }
8c5fd7ef
AD		 5466 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5467 DRM_DEBUG("si_irq_set: sw int dma\n");
5468 dma_cntl |= TRAP_ENABLE;
5469 }
5470
5471 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5472 DRM_DEBUG("si_irq_set: sw int dma1\n");
5473 dma_cntl1 |= TRAP_ENABLE;
5474 }
25a857fb 5475 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 5476 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 5477 DRM_DEBUG("si_irq_set: vblank 0\n");
5478 crtc1 |= VBLANK_INT_MASK;
5479 }
5480 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 5481 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 5482 DRM_DEBUG("si_irq_set: vblank 1\n");
5483 crtc2 |= VBLANK_INT_MASK;
5484 }
5485 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 5486 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 5487 DRM_DEBUG("si_irq_set: vblank 2\n");
5488 crtc3 |= VBLANK_INT_MASK;
5489 }
5490 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 5491 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 5492 DRM_DEBUG("si_irq_set: vblank 3\n");
5493 crtc4 |= VBLANK_INT_MASK;
5494 }
5495 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 5496 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 5497 DRM_DEBUG("si_irq_set: vblank 4\n");
5498 crtc5 |= VBLANK_INT_MASK;
5499 }
5500 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 5501 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 5502 DRM_DEBUG("si_irq_set: vblank 5\n");
5503 crtc6 |= VBLANK_INT_MASK;
5504 }
5505 if (rdev->irq.hpd[0]) {
5506 DRM_DEBUG("si_irq_set: hpd 1\n");
5507 hpd1 |= DC_HPDx_INT_EN;
5508 }
5509 if (rdev->irq.hpd[1]) {
5510 DRM_DEBUG("si_irq_set: hpd 2\n");
5511 hpd2 |= DC_HPDx_INT_EN;
5512 }
5513 if (rdev->irq.hpd[2]) {
5514 DRM_DEBUG("si_irq_set: hpd 3\n");
5515 hpd3 |= DC_HPDx_INT_EN;
5516 }
5517 if (rdev->irq.hpd[3]) {
5518 DRM_DEBUG("si_irq_set: hpd 4\n");
5519 hpd4 |= DC_HPDx_INT_EN;
5520 }
5521 if (rdev->irq.hpd[4]) {
5522 DRM_DEBUG("si_irq_set: hpd 5\n");
5523 hpd5 |= DC_HPDx_INT_EN;
5524 }
5525 if (rdev->irq.hpd[5]) {
5526 DRM_DEBUG("si_irq_set: hpd 6\n");
5527 hpd6 |= DC_HPDx_INT_EN;
5528 }
25a857fb
AD		 5529
5530 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5531 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5532 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5533
8c5fd7ef
AD		 5534 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5535 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5536
25a857fb
AD		 5537 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5538
5153550a
AD		 5539 if (rdev->num_crtc >= 2) {
5540 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5541 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5542 }
25a857fb
AD		 5543 if (rdev->num_crtc >= 4) {
5544 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5545 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5546 }
5547 if (rdev->num_crtc >= 6) {
5548 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5549 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5550 }
5551
5153550a
AD		 5552 if (rdev->num_crtc >= 2) {
5553 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5554 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5555 }
25a857fb
AD		 5556 if (rdev->num_crtc >= 4) {
5557 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5558 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5559 }
5560 if (rdev->num_crtc >= 6) {
5561 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5562 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5563 }
5564
5153550a
AD		 5565 if (!ASIC_IS_NODCE(rdev)) {
5566 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5567 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5568 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5569 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5570 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5571 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5572 }
25a857fb
AD		 5573
5574 return 0;
5575}
5576
5577static inline void si_irq_ack(struct radeon_device *rdev)
5578{
5579 u32 tmp;
5580
5153550a
AD		 5581 if (ASIC_IS_NODCE(rdev))
5582 return;
5583
25a857fb
AD		 5584 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5585 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5586 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5587 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5588 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5589 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5590 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5591 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5592 if (rdev->num_crtc >= 4) {
5593 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5594 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5595 }
5596 if (rdev->num_crtc >= 6) {
5597 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5598 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5599 }
5600
5601 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5602 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5603 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5604 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5605 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5606 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5607 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5608 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5609 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5610 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5611 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5612 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5613
5614 if (rdev->num_crtc >= 4) {
5615 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5616 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5617 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5618 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5619 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5620 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5621 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5622 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5623 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5624 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5625 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5626 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5627 }
5628
5629 if (rdev->num_crtc >= 6) {
5630 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5631 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5632 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5633 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5634 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5635 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5636 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5637 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5638 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5639 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5640 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5641 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5642 }
5643
5644 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5645 tmp = RREG32(DC_HPD1_INT_CONTROL);
5646 tmp |= DC_HPDx_INT_ACK;
5647 WREG32(DC_HPD1_INT_CONTROL, tmp);
5648 }
5649 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5650 tmp = RREG32(DC_HPD2_INT_CONTROL);
5651 tmp |= DC_HPDx_INT_ACK;
5652 WREG32(DC_HPD2_INT_CONTROL, tmp);
5653 }
5654 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5655 tmp = RREG32(DC_HPD3_INT_CONTROL);
5656 tmp |= DC_HPDx_INT_ACK;
5657 WREG32(DC_HPD3_INT_CONTROL, tmp);
5658 }
5659 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5660 tmp = RREG32(DC_HPD4_INT_CONTROL);
5661 tmp |= DC_HPDx_INT_ACK;
5662 WREG32(DC_HPD4_INT_CONTROL, tmp);
5663 }
5664 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5665 tmp = RREG32(DC_HPD5_INT_CONTROL);
5666 tmp |= DC_HPDx_INT_ACK;
5667 WREG32(DC_HPD5_INT_CONTROL, tmp);
5668 }
5669 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5670 tmp = RREG32(DC_HPD5_INT_CONTROL);
5671 tmp |= DC_HPDx_INT_ACK;
5672 WREG32(DC_HPD6_INT_CONTROL, tmp);
5673 }
5674}
5675
5676static void si_irq_disable(struct radeon_device *rdev)
5677{
5678 si_disable_interrupts(rdev);
5679 /* Wait and acknowledge irq */
5680 mdelay(1);
5681 si_irq_ack(rdev);
5682 si_disable_interrupt_state(rdev);
5683}
5684
5685static void si_irq_suspend(struct radeon_device *rdev)
5686{
5687 si_irq_disable(rdev);
5688 si_rlc_stop(rdev);
5689}
5690
9b136d51
AD		 5691static void si_irq_fini(struct radeon_device *rdev)
5692{
5693 si_irq_suspend(rdev);
5694 r600_ih_ring_fini(rdev);
5695}
5696
25a857fb
AD		 5697static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
5698{
5699 u32 wptr, tmp;
5700
5701 if (rdev->wb.enabled)
5702 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5703 else
5704 wptr = RREG32(IH_RB_WPTR);
5705
5706 if (wptr & RB_OVERFLOW) {
5707 /* When a ring buffer overflow happen start parsing interrupt
5708 * from the last not overwritten vector (wptr + 16). Hopefully
5709 * this should allow us to catchup.
5710 */
5711 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5712 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
5713 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5714 tmp = RREG32(IH_RB_CNTL);
5715 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5716 WREG32(IH_RB_CNTL, tmp);
5717 }
5718 return (wptr & rdev->ih.ptr_mask);
5719}
5720
5721/* SI IV Ring
5722 * Each IV ring entry is 128 bits:
5723 * [7:0] - interrupt source id
5724 * [31:8] - reserved
5725 * [59:32] - interrupt source data
5726 * [63:60] - reserved
5727 * [71:64] - RINGID
5728 * [79:72] - VMID
5729 * [127:80] - reserved
5730 */
5731int si_irq_process(struct radeon_device *rdev)
5732{
5733 u32 wptr;
5734 u32 rptr;
5735 u32 src_id, src_data, ring_id;
5736 u32 ring_index;
25a857fb
AD		 5737 bool queue_hotplug = false;
5738
5739 if (!rdev->ih.enabled || rdev->shutdown)
5740 return IRQ_NONE;
5741
5742 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 5743
5744restart_ih:
5745 /* is somebody else already processing irqs? */
5746 if (atomic_xchg(&rdev->ih.lock, 1))
5747 return IRQ_NONE;
5748
25a857fb
AD		 5749 rptr = rdev->ih.rptr;
5750 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
5751
25a857fb
AD		 5752 /* Order reading of wptr vs. reading of IH ring data */
5753 rmb();
5754
5755 /* display interrupts */
5756 si_irq_ack(rdev);
5757
25a857fb
AD		 5758 while (rptr != wptr) {
5759 /* wptr/rptr are in bytes! */
5760 ring_index = rptr / 4;
5761 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
5762 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
5763 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
5764
5765 switch (src_id) {
5766 case 1: /* D1 vblank/vline */
5767 switch (src_data) {
5768 case 0: /* D1 vblank */
5769 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
5770 if (rdev->irq.crtc_vblank_int[0]) {
5771 drm_handle_vblank(rdev->ddev, 0);
5772 rdev->pm.vblank_sync = true;
5773 wake_up(&rdev->irq.vblank_queue);
5774 }
736fc37f 5775 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 5776 radeon_crtc_handle_flip(rdev, 0);
5777 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
5778 DRM_DEBUG("IH: D1 vblank\n");
5779 }
5780 break;
5781 case 1: /* D1 vline */
5782 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
5783 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
5784 DRM_DEBUG("IH: D1 vline\n");
5785 }
5786 break;
5787 default:
5788 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5789 break;
5790 }
5791 break;
5792 case 2: /* D2 vblank/vline */
5793 switch (src_data) {
5794 case 0: /* D2 vblank */
5795 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
5796 if (rdev->irq.crtc_vblank_int[1]) {
5797 drm_handle_vblank(rdev->ddev, 1);
5798 rdev->pm.vblank_sync = true;
5799 wake_up(&rdev->irq.vblank_queue);
5800 }
736fc37f 5801 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 5802 radeon_crtc_handle_flip(rdev, 1);
5803 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5804 DRM_DEBUG("IH: D2 vblank\n");
5805 }
5806 break;
5807 case 1: /* D2 vline */
5808 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
5809 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5810 DRM_DEBUG("IH: D2 vline\n");
5811 }
5812 break;
5813 default:
5814 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5815 break;
5816 }
5817 break;
5818 case 3: /* D3 vblank/vline */
5819 switch (src_data) {
5820 case 0: /* D3 vblank */
5821 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
5822 if (rdev->irq.crtc_vblank_int[2]) {
5823 drm_handle_vblank(rdev->ddev, 2);
5824 rdev->pm.vblank_sync = true;
5825 wake_up(&rdev->irq.vblank_queue);
5826 }
736fc37f 5827 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 5828 radeon_crtc_handle_flip(rdev, 2);
5829 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5830 DRM_DEBUG("IH: D3 vblank\n");
5831 }
5832 break;
5833 case 1: /* D3 vline */
5834 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
5835 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5836 DRM_DEBUG("IH: D3 vline\n");
5837 }
5838 break;
5839 default:
5840 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5841 break;
5842 }
5843 break;
5844 case 4: /* D4 vblank/vline */
5845 switch (src_data) {
5846 case 0: /* D4 vblank */
5847 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
5848 if (rdev->irq.crtc_vblank_int[3]) {
5849 drm_handle_vblank(rdev->ddev, 3);
5850 rdev->pm.vblank_sync = true;
5851 wake_up(&rdev->irq.vblank_queue);
5852 }
736fc37f 5853 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 5854 radeon_crtc_handle_flip(rdev, 3);
5855 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5856 DRM_DEBUG("IH: D4 vblank\n");
5857 }
5858 break;
5859 case 1: /* D4 vline */
5860 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
5861 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5862 DRM_DEBUG("IH: D4 vline\n");
5863 }
5864 break;
5865 default:
5866 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5867 break;
5868 }
5869 break;
5870 case 5: /* D5 vblank/vline */
5871 switch (src_data) {
5872 case 0: /* D5 vblank */
5873 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
5874 if (rdev->irq.crtc_vblank_int[4]) {
5875 drm_handle_vblank(rdev->ddev, 4);
5876 rdev->pm.vblank_sync = true;
5877 wake_up(&rdev->irq.vblank_queue);
5878 }
736fc37f 5879 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 5880 radeon_crtc_handle_flip(rdev, 4);
5881 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5882 DRM_DEBUG("IH: D5 vblank\n");
5883 }
5884 break;
5885 case 1: /* D5 vline */
5886 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
5887 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5888 DRM_DEBUG("IH: D5 vline\n");
5889 }
5890 break;
5891 default:
5892 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5893 break;
5894 }
5895 break;
5896 case 6: /* D6 vblank/vline */
5897 switch (src_data) {
5898 case 0: /* D6 vblank */
5899 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
5900 if (rdev->irq.crtc_vblank_int[5]) {
5901 drm_handle_vblank(rdev->ddev, 5);
5902 rdev->pm.vblank_sync = true;
5903 wake_up(&rdev->irq.vblank_queue);
5904 }
736fc37f 5905 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 5906 radeon_crtc_handle_flip(rdev, 5);
5907 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5908 DRM_DEBUG("IH: D6 vblank\n");
5909 }
5910 break;
5911 case 1: /* D6 vline */
5912 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
5913 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5914 DRM_DEBUG("IH: D6 vline\n");
5915 }
5916 break;
5917 default:
5918 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5919 break;
5920 }
5921 break;
5922 case 42: /* HPD hotplug */
5923 switch (src_data) {
5924 case 0:
5925 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5926 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5927 queue_hotplug = true;
5928 DRM_DEBUG("IH: HPD1\n");
5929 }
5930 break;
5931 case 1:
5932 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5933 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5934 queue_hotplug = true;
5935 DRM_DEBUG("IH: HPD2\n");
5936 }
5937 break;
5938 case 2:
5939 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5940 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5941 queue_hotplug = true;
5942 DRM_DEBUG("IH: HPD3\n");
5943 }
5944 break;
5945 case 3:
5946 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5947 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5948 queue_hotplug = true;
5949 DRM_DEBUG("IH: HPD4\n");
5950 }
5951 break;
5952 case 4:
5953 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5954 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5955 queue_hotplug = true;
5956 DRM_DEBUG("IH: HPD5\n");
5957 }
5958 break;
5959 case 5:
5960 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5961 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5962 queue_hotplug = true;
5963 DRM_DEBUG("IH: HPD6\n");
5964 }
5965 break;
5966 default:
5967 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5968 break;
5969 }
5970 break;
ae133a11
CK		 5971 case 146:
5972 case 147:
5973 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5974 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5975 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5976 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5977 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5978 /* reset addr and status */
5979 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5980 break;
25a857fb
AD		 5981 case 176: /* RINGID0 CP_INT */
5982 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5983 break;
5984 case 177: /* RINGID1 CP_INT */
5985 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5986 break;
5987 case 178: /* RINGID2 CP_INT */
5988 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5989 break;
5990 case 181: /* CP EOP event */
5991 DRM_DEBUG("IH: CP EOP\n");
5992 switch (ring_id) {
5993 case 0:
5994 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5995 break;
5996 case 1:
5997 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5998 break;
5999 case 2:
6000 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6001 break;
6002 }
6003 break;
8c5fd7ef
AD		 6004 case 224: /* DMA trap event */
6005 DRM_DEBUG("IH: DMA trap\n");
6006 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6007 break;
25a857fb
AD		 6008 case 233: /* GUI IDLE */
6009 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6010 break;
8c5fd7ef
AD		 6011 case 244: /* DMA trap event */
6012 DRM_DEBUG("IH: DMA1 trap\n");
6013 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6014 break;
25a857fb
AD		 6015 default:
6016 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6017 break;
6018 }
6019
6020 /* wptr/rptr are in bytes! */
6021 rptr += 16;
6022 rptr &= rdev->ih.ptr_mask;
6023 }
25a857fb
AD		 6024 if (queue_hotplug)
6025 schedule_work(&rdev->hotplug_work);
6026 rdev->ih.rptr = rptr;
6027 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 6028 atomic_set(&rdev->ih.lock, 0);
6029
6030 /* make sure wptr hasn't changed while processing */
6031 wptr = si_get_ih_wptr(rdev);
6032 if (wptr != rptr)
6033 goto restart_ih;
6034
25a857fb
AD		 6035 return IRQ_HANDLED;
6036}
6037
8c5fd7ef
AD		 6038/**
6039 * si_copy_dma - copy pages using the DMA engine
6040 *
6041 * @rdev: radeon_device pointer
6042 * @src_offset: src GPU address
6043 * @dst_offset: dst GPU address
6044 * @num_gpu_pages: number of GPU pages to xfer
6045 * @fence: radeon fence object
6046 *
6047 * Copy GPU paging using the DMA engine (SI).
6048 * Used by the radeon ttm implementation to move pages if
6049 * registered as the asic copy callback.
6050 */
6051int si_copy_dma(struct radeon_device *rdev,
6052 uint64_t src_offset, uint64_t dst_offset,
6053 unsigned num_gpu_pages,
6054 struct radeon_fence **fence)
6055{
6056 struct radeon_semaphore *sem = NULL;
6057 int ring_index = rdev->asic->copy.dma_ring_index;
6058 struct radeon_ring *ring = &rdev->ring[ring_index];
6059 u32 size_in_bytes, cur_size_in_bytes;
6060 int i, num_loops;
6061 int r = 0;
6062
6063 r = radeon_semaphore_create(rdev, &sem);
6064 if (r) {
6065 DRM_ERROR("radeon: moving bo (%d).\n", r);
6066 return r;
6067 }
6068
6069 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
6070 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
6071 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
6072 if (r) {
6073 DRM_ERROR("radeon: moving bo (%d).\n", r);
6074 radeon_semaphore_free(rdev, &sem, NULL);
6075 return r;
6076 }
6077
6078 if (radeon_fence_need_sync(*fence, ring->idx)) {
6079 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
6080 ring->idx);
6081 radeon_fence_note_sync(*fence, ring->idx);
6082 } else {
6083 radeon_semaphore_free(rdev, &sem, NULL);
6084 }
6085
6086 for (i = 0; i < num_loops; i++) {
6087 cur_size_in_bytes = size_in_bytes;
6088 if (cur_size_in_bytes > 0xFFFFF)
6089 cur_size_in_bytes = 0xFFFFF;
6090 size_in_bytes -= cur_size_in_bytes;
6091 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
6092 radeon_ring_write(ring, dst_offset & 0xffffffff);
6093 radeon_ring_write(ring, src_offset & 0xffffffff);
6094 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
6095 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
6096 src_offset += cur_size_in_bytes;
6097 dst_offset += cur_size_in_bytes;
6098 }
6099
6100 r = radeon_fence_emit(rdev, fence, ring->idx);
6101 if (r) {
6102 radeon_ring_unlock_undo(rdev, ring);
6103 return r;
6104 }
6105
6106 radeon_ring_unlock_commit(rdev, ring);
6107 radeon_semaphore_free(rdev, &sem, *fence);
6108
6109 return r;
6110}
6111
9b136d51
AD		 6112/*
6113 * startup/shutdown callbacks
6114 */
6115static int si_startup(struct radeon_device *rdev)
6116{
6117 struct radeon_ring *ring;
6118 int r;
6119
b9d305df
AD		 6120 /* enable pcie gen2/3 link */
6121 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6122 /* enable aspm */
6123 si_program_aspm(rdev);
b9d305df 6124
9b136d51
AD		 6125 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6126 !rdev->rlc_fw || !rdev->mc_fw) {
6127 r = si_init_microcode(rdev);
6128 if (r) {
6129 DRM_ERROR("Failed to load firmware!\n");
6130 return r;
6131 }
6132 }
6133
6134 r = si_mc_load_microcode(rdev);
6135 if (r) {
6136 DRM_ERROR("Failed to load MC firmware!\n");
6137 return r;
6138 }
6139
6140 r = r600_vram_scratch_init(rdev);
6141 if (r)
6142 return r;
6143
6144 si_mc_program(rdev);
6145 r = si_pcie_gart_enable(rdev);
6146 if (r)
6147 return r;
6148 si_gpu_init(rdev);
6149
9b136d51
AD		 6150 /* allocate rlc buffers */
6151 r = si_rlc_init(rdev);
6152 if (r) {
6153 DRM_ERROR("Failed to init rlc BOs!\n");
6154 return r;
6155 }
6156
6157 /* allocate wb buffer */
6158 r = radeon_wb_init(rdev);
6159 if (r)
6160 return r;
6161
6162 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6163 if (r) {
6164 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6165 return r;
6166 }
6167
6168 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6169 if (r) {
6170 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6171 return r;
6172 }
6173
6174 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6175 if (r) {
6176 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6177 return r;
6178 }
6179
8c5fd7ef
AD		 6180 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6181 if (r) {
6182 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6183 return r;
6184 }
6185
6186 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6187 if (r) {
6188 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6189 return r;
6190 }
6191
1df0d523
AD		 6192 if (rdev->has_uvd) {
6193 r = rv770_uvd_resume(rdev);
6194 if (!r) {
6195 r = radeon_fence_driver_start_ring(rdev,
6196 R600_RING_TYPE_UVD_INDEX);
6197 if (r)
6198 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6199 }
f2ba57b5 6200 if (r)
1df0d523 6201 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 6202 }
f2ba57b5 6203
9b136d51 6204 /* Enable IRQ */
e49f3959
AH		 6205 if (!rdev->irq.installed) {
6206 r = radeon_irq_kms_init(rdev);
6207 if (r)
6208 return r;
6209 }
6210
9b136d51
AD		 6211 r = si_irq_init(rdev);
6212 if (r) {
6213 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6214 radeon_irq_kms_fini(rdev);
6215 return r;
6216 }
6217 si_irq_set(rdev);
6218
6219 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6220 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6221 CP_RB0_RPTR, CP_RB0_WPTR,
6222 0, 0xfffff, RADEON_CP_PACKET2);
6223 if (r)
6224 return r;
6225
6226 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6227 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6228 CP_RB1_RPTR, CP_RB1_WPTR,
6229 0, 0xfffff, RADEON_CP_PACKET2);
6230 if (r)
6231 return r;
6232
6233 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6234 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6235 CP_RB2_RPTR, CP_RB2_WPTR,
6236 0, 0xfffff, RADEON_CP_PACKET2);
6237 if (r)
6238 return r;
6239
8c5fd7ef
AD		 6240 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6241 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6242 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6243 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
6244 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6245 if (r)
6246 return r;
6247
6248 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6249 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6250 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6251 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
6252 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6253 if (r)
6254 return r;
6255
9b136d51
AD		 6256 r = si_cp_load_microcode(rdev);
6257 if (r)
6258 return r;
6259 r = si_cp_resume(rdev);
6260 if (r)
6261 return r;
6262
8c5fd7ef
AD		 6263 r = cayman_dma_resume(rdev);
6264 if (r)
6265 return r;
6266
1df0d523
AD		 6267 if (rdev->has_uvd) {
6268 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6269 if (ring->ring_size) {
6270 r = radeon_ring_init(rdev, ring, ring->ring_size,
6271 R600_WB_UVD_RPTR_OFFSET,
6272 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6273 0, 0xfffff, RADEON_CP_PACKET2);
6274 if (!r)
6275 r = r600_uvd_init(rdev);
6276 if (r)
6277 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6278 }
f2ba57b5
CK		 6279 }
6280
2898c348
CK		 6281 r = radeon_ib_pool_init(rdev);
6282 if (r) {
6283 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6284 return r;
2898c348 6285 }
9b136d51 6286
c6105f24
CK		 6287 r = radeon_vm_manager_init(rdev);
6288 if (r) {
6289 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6290 return r;
c6105f24 6291 }
9b136d51
AD		 6292
6293 return 0;
6294}
6295
6296int si_resume(struct radeon_device *rdev)
6297{
6298 int r;
6299
6300 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6301 * posting will perform necessary task to bring back GPU into good
6302 * shape.
6303 */
6304 /* post card */
6305 atom_asic_init(rdev->mode_info.atom_context);
6306
205996c0
AD		 6307 /* init golden registers */
6308 si_init_golden_registers(rdev);
6309
9b136d51
AD		 6310 rdev->accel_working = true;
6311 r = si_startup(rdev);
6312 if (r) {
6313 DRM_ERROR("si startup failed on resume\n");
6314 rdev->accel_working = false;
6315 return r;
6316 }
6317
6318 return r;
6319
6320}
6321
6322int si_suspend(struct radeon_device *rdev)
6323{
fa3daf9a 6324 radeon_vm_manager_fini(rdev);
9b136d51 6325 si_cp_enable(rdev, false);
8c5fd7ef 6326 cayman_dma_stop(rdev);
1df0d523
AD		 6327 if (rdev->has_uvd) {
6328 r600_uvd_rbc_stop(rdev);
6329 radeon_uvd_suspend(rdev);
6330 }
9b136d51
AD		 6331 si_irq_suspend(rdev);
6332 radeon_wb_disable(rdev);
6333 si_pcie_gart_disable(rdev);
6334 return 0;
6335}
6336
6337/* Plan is to move initialization in that function and use
6338 * helper function so that radeon_device_init pretty much
6339 * do nothing more than calling asic specific function. This
6340 * should also allow to remove a bunch of callback function
6341 * like vram_info.
6342 */
6343int si_init(struct radeon_device *rdev)
6344{
6345 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6346 int r;
6347
9b136d51
AD		 6348 /* Read BIOS */
6349 if (!radeon_get_bios(rdev)) {
6350 if (ASIC_IS_AVIVO(rdev))
6351 return -EINVAL;
6352 }
6353 /* Must be an ATOMBIOS */
6354 if (!rdev->is_atom_bios) {
6355 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6356 return -EINVAL;
6357 }
6358 r = radeon_atombios_init(rdev);
6359 if (r)
6360 return r;
6361
6362 /* Post card if necessary */
6363 if (!radeon_card_posted(rdev)) {
6364 if (!rdev->bios) {
6365 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6366 return -EINVAL;
6367 }
6368 DRM_INFO("GPU not posted. posting now...\n");
6369 atom_asic_init(rdev->mode_info.atom_context);
6370 }
205996c0
AD		 6371 /* init golden registers */
6372 si_init_golden_registers(rdev);
9b136d51
AD		 6373 /* Initialize scratch registers */
6374 si_scratch_init(rdev);
6375 /* Initialize surface registers */
6376 radeon_surface_init(rdev);
6377 /* Initialize clocks */
6378 radeon_get_clock_info(rdev->ddev);
6379
6380 /* Fence driver */
6381 r = radeon_fence_driver_init(rdev);
6382 if (r)
6383 return r;
6384
6385 /* initialize memory controller */
6386 r = si_mc_init(rdev);
6387 if (r)
6388 return r;
6389 /* Memory manager */
6390 r = radeon_bo_init(rdev);
6391 if (r)
6392 return r;
6393
9b136d51
AD		 6394 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6395 ring->ring_obj = NULL;
6396 r600_ring_init(rdev, ring, 1024 * 1024);
6397
6398 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6399 ring->ring_obj = NULL;
6400 r600_ring_init(rdev, ring, 1024 * 1024);
6401
6402 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6403 ring->ring_obj = NULL;
6404 r600_ring_init(rdev, ring, 1024 * 1024);
6405
8c5fd7ef
AD		 6406 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6407 ring->ring_obj = NULL;
6408 r600_ring_init(rdev, ring, 64 * 1024);
6409
6410 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6411 ring->ring_obj = NULL;
6412 r600_ring_init(rdev, ring, 64 * 1024);
6413
1df0d523
AD		 6414 if (rdev->has_uvd) {
6415 r = radeon_uvd_init(rdev);
6416 if (!r) {
6417 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6418 ring->ring_obj = NULL;
6419 r600_ring_init(rdev, ring, 4096);
6420 }
f2ba57b5
CK		 6421 }
6422
9b136d51
AD		 6423 rdev->ih.ring_obj = NULL;
6424 r600_ih_ring_init(rdev, 64 * 1024);
6425
6426 r = r600_pcie_gart_init(rdev);
6427 if (r)
6428 return r;
6429
9b136d51 6430 rdev->accel_working = true;
9b136d51
AD		 6431 r = si_startup(rdev);
6432 if (r) {
6433 dev_err(rdev->dev, "disabling GPU acceleration\n");
6434 si_cp_fini(rdev);
8c5fd7ef 6435 cayman_dma_fini(rdev);
9b136d51
AD		 6436 si_irq_fini(rdev);
6437 si_rlc_fini(rdev);
6438 radeon_wb_fini(rdev);
2898c348 6439 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6440 radeon_vm_manager_fini(rdev);
6441 radeon_irq_kms_fini(rdev);
6442 si_pcie_gart_fini(rdev);
6443 rdev->accel_working = false;
6444 }
6445
6446 /* Don't start up if the MC ucode is missing.
6447 * The default clocks and voltages before the MC ucode
6448 * is loaded are not suffient for advanced operations.
6449 */
6450 if (!rdev->mc_fw) {
6451 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6452 return -EINVAL;
6453 }
6454
6455 return 0;
6456}
6457
6458void si_fini(struct radeon_device *rdev)
6459{
9b136d51 6460 si_cp_fini(rdev);
8c5fd7ef 6461 cayman_dma_fini(rdev);
9b136d51
AD		 6462 si_irq_fini(rdev);
6463 si_rlc_fini(rdev);
f8f84ac5
AD		 6464 si_fini_cg(rdev);
6465 si_fini_pg(rdev);
9b136d51
AD		 6466 radeon_wb_fini(rdev);
6467 radeon_vm_manager_fini(rdev);
2898c348 6468 radeon_ib_pool_fini(rdev);
9b136d51 6469 radeon_irq_kms_fini(rdev);
1df0d523
AD		 6470 if (rdev->has_uvd)
6471 radeon_uvd_fini(rdev);
9b136d51
AD		 6472 si_pcie_gart_fini(rdev);
6473 r600_vram_scratch_fini(rdev);
6474 radeon_gem_fini(rdev);
9b136d51
AD		 6475 radeon_fence_driver_fini(rdev);
6476 radeon_bo_fini(rdev);
6477 radeon_atombios_fini(rdev);
6478 kfree(rdev->bios);
6479 rdev->bios = NULL;
6480}
6481
6759a0a7 6482/**
d0418894 6483 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6484 *
6485 * @rdev: radeon_device pointer
6486 *
6487 * Fetches a GPU clock counter snapshot (SI).
6488 * Returns the 64 bit clock counter snapshot.
6489 */
d0418894 6490uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6491{
6492 uint64_t clock;
6493
6494 mutex_lock(&rdev->gpu_clock_mutex);
6495 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6496 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6497 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6498 mutex_unlock(&rdev->gpu_clock_mutex);
6499 return clock;
6500}
2539eb02 6501
2539eb02
CK		 6502int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6503{
facd112d 6504 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6505 int r;
6506
4ed10835
CK		 6507 /* bypass vclk and dclk with bclk */
6508 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6509 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6510 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6511
6512 /* put PLL in bypass mode */
6513 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6514
6515 if (!vclk || !dclk) {
6516 /* keep the Bypass mode, put PLL to sleep */
6517 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6518 return 0;
6519 }
6520
facd112d
CK		 6521 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6522 16384, 0x03FFFFFF, 0, 128, 5,
6523 &fb_div, &vclk_div, &dclk_div);
6524 if (r)
6525 return r;
2539eb02
CK		 6526
6527 /* set RESET_ANTI_MUX to 0 */
6528 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6529
6530 /* set VCO_MODE to 1 */
6531 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6532
6533 /* toggle UPLL_SLEEP to 1 then back to 0 */
6534 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6535 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6536
6537 /* deassert UPLL_RESET */
6538 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6539
6540 mdelay(1);
6541
facd112d 6542 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6543 if (r)
6544 return r;
6545
6546 /* assert UPLL_RESET again */
6547 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6548
6549 /* disable spread spectrum. */
6550 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6551
6552 /* set feedback divider */
facd112d 6553 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 6554
6555 /* set ref divider to 0 */
6556 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6557
facd112d 6558 if (fb_div < 307200)
2539eb02
CK		 6559 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6560 else
6561 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6562
6563 /* set PDIV_A and PDIV_B */
6564 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 6565 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 6566 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6567
6568 /* give the PLL some time to settle */
6569 mdelay(15);
6570
6571 /* deassert PLL_RESET */
6572 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6573
6574 mdelay(15);
6575
6576 /* switch from bypass mode to normal mode */
6577 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6578
facd112d 6579 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6580 if (r)
6581 return r;
6582
6583 /* switch VCLK and DCLK selection */
6584 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6585 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6586 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6587
6588 mdelay(100);
6589
6590 return 0;
6591}
b9d305df
AD		 6592
6593static void si_pcie_gen3_enable(struct radeon_device *rdev)
6594{
6595 struct pci_dev *root = rdev->pdev->bus->self;
6596 int bridge_pos, gpu_pos;
6597 u32 speed_cntl, mask, current_data_rate;
6598 int ret, i;
6599 u16 tmp16;
6600
6601 if (radeon_pcie_gen2 == 0)
6602 return;
6603
6604 if (rdev->flags & RADEON_IS_IGP)
6605 return;
6606
6607 if (!(rdev->flags & RADEON_IS_PCIE))
6608 return;
6609
6610 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6611 if (ret != 0)
6612 return;
6613
6614 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6615 return;
6616
6617 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6618 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6619 LC_CURRENT_DATA_RATE_SHIFT;
6620 if (mask & DRM_PCIE_SPEED_80) {
6621 if (current_data_rate == 2) {
6622 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6623 return;
6624 }
6625 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6626 } else if (mask & DRM_PCIE_SPEED_50) {
6627 if (current_data_rate == 1) {
6628 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6629 return;
6630 }
6631 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6632 }
6633
6634 bridge_pos = pci_pcie_cap(root);
6635 if (!bridge_pos)
6636 return;
6637
6638 gpu_pos = pci_pcie_cap(rdev->pdev);
6639 if (!gpu_pos)
6640 return;
6641
6642 if (mask & DRM_PCIE_SPEED_80) {
6643 /* re-try equalization if gen3 is not already enabled */
6644 if (current_data_rate != 2) {
6645 u16 bridge_cfg, gpu_cfg;
6646 u16 bridge_cfg2, gpu_cfg2;
6647 u32 max_lw, current_lw, tmp;
6648
6649 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6650 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6651
6652 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6653 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6654
6655 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6656 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6657
6658 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6659 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6660 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6661
6662 if (current_lw < max_lw) {
6663 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6664 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6665 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6666 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6667 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6668 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6669 }
6670 }
6671
6672 for (i = 0; i < 10; i++) {
6673 /* check status */
6674 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6675 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6676 break;
6677
6678 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6679 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6680
6681 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6682 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6683
6684 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6685 tmp |= LC_SET_QUIESCE;
6686 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6687
6688 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6689 tmp |= LC_REDO_EQ;
6690 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6691
6692 mdelay(100);
6693
6694 /* linkctl */
6695 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6696 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6697 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6698 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6699
6700 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6701 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6702 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6703 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6704
6705 /* linkctl2 */
6706 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6707 tmp16 &= ~((1 << 4) | (7 << 9));
6708 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6709 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6710
6711 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6712 tmp16 &= ~((1 << 4) | (7 << 9));
6713 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6714 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6715
6716 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6717 tmp &= ~LC_SET_QUIESCE;
6718 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6719 }
6720 }
6721 }
6722
6723 /* set the link speed */
6724 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6725 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6726 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6727
6728 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6729 tmp16 &= ~0xf;
6730 if (mask & DRM_PCIE_SPEED_80)
6731 tmp16 |= 3; /* gen3 */
6732 else if (mask & DRM_PCIE_SPEED_50)
6733 tmp16 |= 2; /* gen2 */
6734 else
6735 tmp16 |= 1; /* gen1 */
6736 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6737
6738 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6739 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
6740 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6741
6742 for (i = 0; i < rdev->usec_timeout; i++) {
6743 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6744 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
6745 break;
6746 udelay(1);
6747 }
6748}
6749
e0bcf165
AD		 6750static void si_program_aspm(struct radeon_device *rdev)
6751{
6752 u32 data, orig;
6753 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
6754 bool disable_clkreq = false;
6755
6756 if (!(rdev->flags & RADEON_IS_PCIE))
6757 return;
6758
6759 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6760 data &= ~LC_XMIT_N_FTS_MASK;
6761 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
6762 if (orig != data)
6763 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
6764
6765 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
6766 data |= LC_GO_TO_RECOVERY;
6767 if (orig != data)
6768 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
6769
6770 orig = data = RREG32_PCIE(PCIE_P_CNTL);
6771 data |= P_IGNORE_EDB_ERR;
6772 if (orig != data)
6773 WREG32_PCIE(PCIE_P_CNTL, data);
6774
6775 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6776 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
6777 data |= LC_PMI_TO_L1_DIS;
6778 if (!disable_l0s)
6779 data |= LC_L0S_INACTIVITY(7);
6780
6781 if (!disable_l1) {
6782 data |= LC_L1_INACTIVITY(7);
6783 data &= ~LC_PMI_TO_L1_DIS;
6784 if (orig != data)
6785 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6786
6787 if (!disable_plloff_in_l1) {
6788 bool clk_req_support;
6789
6790 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6791 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6792 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6793 if (orig != data)
6794 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6795
6796 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6797 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6798 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6799 if (orig != data)
6800 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6801
6802 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6803 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6804 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6805 if (orig != data)
6806 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6807
6808 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6809 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6810 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6811 if (orig != data)
6812 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6813
6814 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
6815 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6816 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6817 if (orig != data)
6818 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6819
6820 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6821 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6822 if (orig != data)
6823 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6824
6825 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
6826 data &= ~PLL_RAMP_UP_TIME_2_MASK;
6827 if (orig != data)
6828 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
6829
6830 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
6831 data &= ~PLL_RAMP_UP_TIME_3_MASK;
6832 if (orig != data)
6833 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
6834
6835 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6836 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6837 if (orig != data)
6838 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6839
6840 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6841 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6842 if (orig != data)
6843 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6844
6845 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
6846 data &= ~PLL_RAMP_UP_TIME_2_MASK;
6847 if (orig != data)
6848 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
6849
6850 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
6851 data &= ~PLL_RAMP_UP_TIME_3_MASK;
6852 if (orig != data)
6853 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
6854 }
6855 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6856 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
6857 data |= LC_DYN_LANES_PWR_STATE(3);
6858 if (orig != data)
6859 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
6860
6861 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
6862 data &= ~LS2_EXIT_TIME_MASK;
6863 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
6864 data |= LS2_EXIT_TIME(5);
6865 if (orig != data)
6866 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
6867
6868 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
6869 data &= ~LS2_EXIT_TIME_MASK;
6870 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
6871 data |= LS2_EXIT_TIME(5);
6872 if (orig != data)
6873 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
6874
6875 if (!disable_clkreq) {
6876 struct pci_dev *root = rdev->pdev->bus->self;
6877 u32 lnkcap;
6878
6879 clk_req_support = false;
6880 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
6881 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
6882 clk_req_support = true;
6883 } else {
6884 clk_req_support = false;
6885 }
6886
6887 if (clk_req_support) {
6888 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
6889 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
6890 if (orig != data)
6891 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
6892
6893 orig = data = RREG32(THM_CLK_CNTL);
6894 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
6895 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
6896 if (orig != data)
6897 WREG32(THM_CLK_CNTL, data);
6898
6899 orig = data = RREG32(MISC_CLK_CNTL);
6900 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
6901 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
6902 if (orig != data)
6903 WREG32(MISC_CLK_CNTL, data);
6904
6905 orig = data = RREG32(CG_CLKPIN_CNTL);
6906 data &= ~BCLK_AS_XCLK;
6907 if (orig != data)
6908 WREG32(CG_CLKPIN_CNTL, data);
6909
6910 orig = data = RREG32(CG_CLKPIN_CNTL_2);
6911 data &= ~FORCE_BIF_REFCLK_EN;
6912 if (orig != data)
6913 WREG32(CG_CLKPIN_CNTL_2, data);
6914
6915 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
6916 data &= ~MPLL_CLKOUT_SEL_MASK;
6917 data |= MPLL_CLKOUT_SEL(4);
6918 if (orig != data)
6919 WREG32(MPLL_BYPASSCLK_SEL, data);
6920
6921 orig = data = RREG32(SPLL_CNTL_MODE);
6922 data &= ~SPLL_REFCLK_SEL_MASK;
6923 if (orig != data)
6924 WREG32(SPLL_CNTL_MODE, data);
6925 }
6926 }
6927 } else {
6928 if (orig != data)
6929 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6930 }
6931
6932 orig = data = RREG32_PCIE(PCIE_CNTL2);
6933 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
6934 if (orig != data)
6935 WREG32_PCIE(PCIE_CNTL2, data);
6936
6937 if (!disable_l0s) {
6938 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6939 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
6940 data = RREG32_PCIE(PCIE_LC_STATUS1);
6941 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
6942 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6943 data &= ~LC_L0S_INACTIVITY_MASK;
6944 if (orig != data)
6945 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6946 }
6947 }
6948 }
6949}
Linux 6.x block layer and io_uring tree(s)