projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: convert SI,CIK to use sumo_rlc functions
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c 24#include <linux/firmware.h>
0f0de06c
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
760285e7 27#include <drm/drmP.h>
43b3cd99
AD		 28#include "radeon.h"
29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
43b3cd99
AD		 31#include "sid.h"
32#include "atom.h"
48c0c902 33#include "si_blit_shaders.h"
bd8cd539 34#include "clearstate_si.h"
a0ceada6 35#include "radeon_ucode.h"
43b3cd99 36
0f0de06c
AD		 37
38MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
a9e61410 43MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
0f0de06c
AD		 44MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
a9e61410 49MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
0f0de06c
AD		 50MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
51MODULE_FIRMWARE("radeon/VERDE_me.bin");
52MODULE_FIRMWARE("radeon/VERDE_ce.bin");
53MODULE_FIRMWARE("radeon/VERDE_mc.bin");
54MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
a9e61410 55MODULE_FIRMWARE("radeon/VERDE_smc.bin");
bcc7f5d2
AD		 56MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
57MODULE_FIRMWARE("radeon/OLAND_me.bin");
58MODULE_FIRMWARE("radeon/OLAND_ce.bin");
59MODULE_FIRMWARE("radeon/OLAND_mc.bin");
60MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
a9e61410 61MODULE_FIRMWARE("radeon/OLAND_smc.bin");
c04c00b4
AD		 62MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
63MODULE_FIRMWARE("radeon/HAINAN_me.bin");
64MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
65MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
66MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
a9e61410 67MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
0f0de06c 68
b9d305df 69static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 70static void si_program_aspm(struct radeon_device *rdev);
1fd11777
AD		 71extern void sumo_rlc_fini(struct radeon_device *rdev);
72extern int sumo_rlc_init(struct radeon_device *rdev);
25a857fb
AD		 73extern int r600_ih_ring_alloc(struct radeon_device *rdev);
74extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 75extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 76extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
77extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 78extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 79extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 80extern bool evergreen_is_display_hung(struct radeon_device *rdev);
0a96d72b 81
6d8cf000
AD		 82static const u32 verde_rlc_save_restore_register_list[] =
83{
84 (0x8000 << 16) | (0x98f4 >> 2),
85 0x00000000,
86 (0x8040 << 16) | (0x98f4 >> 2),
87 0x00000000,
88 (0x8000 << 16) | (0xe80 >> 2),
89 0x00000000,
90 (0x8040 << 16) | (0xe80 >> 2),
91 0x00000000,
92 (0x8000 << 16) | (0x89bc >> 2),
93 0x00000000,
94 (0x8040 << 16) | (0x89bc >> 2),
95 0x00000000,
96 (0x8000 << 16) | (0x8c1c >> 2),
97 0x00000000,
98 (0x8040 << 16) | (0x8c1c >> 2),
99 0x00000000,
100 (0x9c00 << 16) | (0x98f0 >> 2),
101 0x00000000,
102 (0x9c00 << 16) | (0xe7c >> 2),
103 0x00000000,
104 (0x8000 << 16) | (0x9148 >> 2),
105 0x00000000,
106 (0x8040 << 16) | (0x9148 >> 2),
107 0x00000000,
108 (0x9c00 << 16) | (0x9150 >> 2),
109 0x00000000,
110 (0x9c00 << 16) | (0x897c >> 2),
111 0x00000000,
112 (0x9c00 << 16) | (0x8d8c >> 2),
113 0x00000000,
114 (0x9c00 << 16) | (0xac54 >> 2),
115 0X00000000,
116 0x3,
117 (0x9c00 << 16) | (0x98f8 >> 2),
118 0x00000000,
119 (0x9c00 << 16) | (0x9910 >> 2),
120 0x00000000,
121 (0x9c00 << 16) | (0x9914 >> 2),
122 0x00000000,
123 (0x9c00 << 16) | (0x9918 >> 2),
124 0x00000000,
125 (0x9c00 << 16) | (0x991c >> 2),
126 0x00000000,
127 (0x9c00 << 16) | (0x9920 >> 2),
128 0x00000000,
129 (0x9c00 << 16) | (0x9924 >> 2),
130 0x00000000,
131 (0x9c00 << 16) | (0x9928 >> 2),
132 0x00000000,
133 (0x9c00 << 16) | (0x992c >> 2),
134 0x00000000,
135 (0x9c00 << 16) | (0x9930 >> 2),
136 0x00000000,
137 (0x9c00 << 16) | (0x9934 >> 2),
138 0x00000000,
139 (0x9c00 << 16) | (0x9938 >> 2),
140 0x00000000,
141 (0x9c00 << 16) | (0x993c >> 2),
142 0x00000000,
143 (0x9c00 << 16) | (0x9940 >> 2),
144 0x00000000,
145 (0x9c00 << 16) | (0x9944 >> 2),
146 0x00000000,
147 (0x9c00 << 16) | (0x9948 >> 2),
148 0x00000000,
149 (0x9c00 << 16) | (0x994c >> 2),
150 0x00000000,
151 (0x9c00 << 16) | (0x9950 >> 2),
152 0x00000000,
153 (0x9c00 << 16) | (0x9954 >> 2),
154 0x00000000,
155 (0x9c00 << 16) | (0x9958 >> 2),
156 0x00000000,
157 (0x9c00 << 16) | (0x995c >> 2),
158 0x00000000,
159 (0x9c00 << 16) | (0x9960 >> 2),
160 0x00000000,
161 (0x9c00 << 16) | (0x9964 >> 2),
162 0x00000000,
163 (0x9c00 << 16) | (0x9968 >> 2),
164 0x00000000,
165 (0x9c00 << 16) | (0x996c >> 2),
166 0x00000000,
167 (0x9c00 << 16) | (0x9970 >> 2),
168 0x00000000,
169 (0x9c00 << 16) | (0x9974 >> 2),
170 0x00000000,
171 (0x9c00 << 16) | (0x9978 >> 2),
172 0x00000000,
173 (0x9c00 << 16) | (0x997c >> 2),
174 0x00000000,
175 (0x9c00 << 16) | (0x9980 >> 2),
176 0x00000000,
177 (0x9c00 << 16) | (0x9984 >> 2),
178 0x00000000,
179 (0x9c00 << 16) | (0x9988 >> 2),
180 0x00000000,
181 (0x9c00 << 16) | (0x998c >> 2),
182 0x00000000,
183 (0x9c00 << 16) | (0x8c00 >> 2),
184 0x00000000,
185 (0x9c00 << 16) | (0x8c14 >> 2),
186 0x00000000,
187 (0x9c00 << 16) | (0x8c04 >> 2),
188 0x00000000,
189 (0x9c00 << 16) | (0x8c08 >> 2),
190 0x00000000,
191 (0x8000 << 16) | (0x9b7c >> 2),
192 0x00000000,
193 (0x8040 << 16) | (0x9b7c >> 2),
194 0x00000000,
195 (0x8000 << 16) | (0xe84 >> 2),
196 0x00000000,
197 (0x8040 << 16) | (0xe84 >> 2),
198 0x00000000,
199 (0x8000 << 16) | (0x89c0 >> 2),
200 0x00000000,
201 (0x8040 << 16) | (0x89c0 >> 2),
202 0x00000000,
203 (0x8000 << 16) | (0x914c >> 2),
204 0x00000000,
205 (0x8040 << 16) | (0x914c >> 2),
206 0x00000000,
207 (0x8000 << 16) | (0x8c20 >> 2),
208 0x00000000,
209 (0x8040 << 16) | (0x8c20 >> 2),
210 0x00000000,
211 (0x8000 << 16) | (0x9354 >> 2),
212 0x00000000,
213 (0x8040 << 16) | (0x9354 >> 2),
214 0x00000000,
215 (0x9c00 << 16) | (0x9060 >> 2),
216 0x00000000,
217 (0x9c00 << 16) | (0x9364 >> 2),
218 0x00000000,
219 (0x9c00 << 16) | (0x9100 >> 2),
220 0x00000000,
221 (0x9c00 << 16) | (0x913c >> 2),
222 0x00000000,
223 (0x8000 << 16) | (0x90e0 >> 2),
224 0x00000000,
225 (0x8000 << 16) | (0x90e4 >> 2),
226 0x00000000,
227 (0x8000 << 16) | (0x90e8 >> 2),
228 0x00000000,
229 (0x8040 << 16) | (0x90e0 >> 2),
230 0x00000000,
231 (0x8040 << 16) | (0x90e4 >> 2),
232 0x00000000,
233 (0x8040 << 16) | (0x90e8 >> 2),
234 0x00000000,
235 (0x9c00 << 16) | (0x8bcc >> 2),
236 0x00000000,
237 (0x9c00 << 16) | (0x8b24 >> 2),
238 0x00000000,
239 (0x9c00 << 16) | (0x88c4 >> 2),
240 0x00000000,
241 (0x9c00 << 16) | (0x8e50 >> 2),
242 0x00000000,
243 (0x9c00 << 16) | (0x8c0c >> 2),
244 0x00000000,
245 (0x9c00 << 16) | (0x8e58 >> 2),
246 0x00000000,
247 (0x9c00 << 16) | (0x8e5c >> 2),
248 0x00000000,
249 (0x9c00 << 16) | (0x9508 >> 2),
250 0x00000000,
251 (0x9c00 << 16) | (0x950c >> 2),
252 0x00000000,
253 (0x9c00 << 16) | (0x9494 >> 2),
254 0x00000000,
255 (0x9c00 << 16) | (0xac0c >> 2),
256 0x00000000,
257 (0x9c00 << 16) | (0xac10 >> 2),
258 0x00000000,
259 (0x9c00 << 16) | (0xac14 >> 2),
260 0x00000000,
261 (0x9c00 << 16) | (0xae00 >> 2),
262 0x00000000,
263 (0x9c00 << 16) | (0xac08 >> 2),
264 0x00000000,
265 (0x9c00 << 16) | (0x88d4 >> 2),
266 0x00000000,
267 (0x9c00 << 16) | (0x88c8 >> 2),
268 0x00000000,
269 (0x9c00 << 16) | (0x88cc >> 2),
270 0x00000000,
271 (0x9c00 << 16) | (0x89b0 >> 2),
272 0x00000000,
273 (0x9c00 << 16) | (0x8b10 >> 2),
274 0x00000000,
275 (0x9c00 << 16) | (0x8a14 >> 2),
276 0x00000000,
277 (0x9c00 << 16) | (0x9830 >> 2),
278 0x00000000,
279 (0x9c00 << 16) | (0x9834 >> 2),
280 0x00000000,
281 (0x9c00 << 16) | (0x9838 >> 2),
282 0x00000000,
283 (0x9c00 << 16) | (0x9a10 >> 2),
284 0x00000000,
285 (0x8000 << 16) | (0x9870 >> 2),
286 0x00000000,
287 (0x8000 << 16) | (0x9874 >> 2),
288 0x00000000,
289 (0x8001 << 16) | (0x9870 >> 2),
290 0x00000000,
291 (0x8001 << 16) | (0x9874 >> 2),
292 0x00000000,
293 (0x8040 << 16) | (0x9870 >> 2),
294 0x00000000,
295 (0x8040 << 16) | (0x9874 >> 2),
296 0x00000000,
297 (0x8041 << 16) | (0x9870 >> 2),
298 0x00000000,
299 (0x8041 << 16) | (0x9874 >> 2),
300 0x00000000,
301 0x00000000
302};
303
205996c0
AD		 304static const u32 tahiti_golden_rlc_registers[] =
305{
306 0xc424, 0xffffffff, 0x00601005,
307 0xc47c, 0xffffffff, 0x10104040,
308 0xc488, 0xffffffff, 0x0100000a,
309 0xc314, 0xffffffff, 0x00000800,
310 0xc30c, 0xffffffff, 0x800000f4,
311 0xf4a8, 0xffffffff, 0x00000000
312};
313
314static const u32 tahiti_golden_registers[] =
315{
316 0x9a10, 0x00010000, 0x00018208,
317 0x9830, 0xffffffff, 0x00000000,
318 0x9834, 0xf00fffff, 0x00000400,
319 0x9838, 0x0002021c, 0x00020200,
320 0xc78, 0x00000080, 0x00000000,
321 0xd030, 0x000300c0, 0x00800040,
322 0xd830, 0x000300c0, 0x00800040,
323 0x5bb0, 0x000000f0, 0x00000070,
324 0x5bc0, 0x00200000, 0x50100000,
325 0x7030, 0x31000311, 0x00000011,
326 0x277c, 0x00000003, 0x000007ff,
327 0x240c, 0x000007ff, 0x00000000,
328 0x8a14, 0xf000001f, 0x00000007,
329 0x8b24, 0xffffffff, 0x00ffffff,
330 0x8b10, 0x0000ff0f, 0x00000000,
331 0x28a4c, 0x07ffffff, 0x4e000000,
332 0x28350, 0x3f3f3fff, 0x2a00126a,
333 0x30, 0x000000ff, 0x0040,
334 0x34, 0x00000040, 0x00004040,
335 0x9100, 0x07ffffff, 0x03000000,
336 0x8e88, 0x01ff1f3f, 0x00000000,
337 0x8e84, 0x01ff1f3f, 0x00000000,
338 0x9060, 0x0000007f, 0x00000020,
339 0x9508, 0x00010000, 0x00010000,
340 0xac14, 0x00000200, 0x000002fb,
341 0xac10, 0xffffffff, 0x0000543b,
342 0xac0c, 0xffffffff, 0xa9210876,
343 0x88d0, 0xffffffff, 0x000fff40,
344 0x88d4, 0x0000001f, 0x00000010,
345 0x1410, 0x20000000, 0x20fffed8,
346 0x15c0, 0x000c0fc0, 0x000c0400
347};
348
349static const u32 tahiti_golden_registers2[] =
350{
351 0xc64, 0x00000001, 0x00000001
352};
353
354static const u32 pitcairn_golden_rlc_registers[] =
355{
356 0xc424, 0xffffffff, 0x00601004,
357 0xc47c, 0xffffffff, 0x10102020,
358 0xc488, 0xffffffff, 0x01000020,
359 0xc314, 0xffffffff, 0x00000800,
360 0xc30c, 0xffffffff, 0x800000a4
361};
362
363static const u32 pitcairn_golden_registers[] =
364{
365 0x9a10, 0x00010000, 0x00018208,
366 0x9830, 0xffffffff, 0x00000000,
367 0x9834, 0xf00fffff, 0x00000400,
368 0x9838, 0x0002021c, 0x00020200,
369 0xc78, 0x00000080, 0x00000000,
370 0xd030, 0x000300c0, 0x00800040,
371 0xd830, 0x000300c0, 0x00800040,
372 0x5bb0, 0x000000f0, 0x00000070,
373 0x5bc0, 0x00200000, 0x50100000,
374 0x7030, 0x31000311, 0x00000011,
375 0x2ae4, 0x00073ffe, 0x000022a2,
376 0x240c, 0x000007ff, 0x00000000,
377 0x8a14, 0xf000001f, 0x00000007,
378 0x8b24, 0xffffffff, 0x00ffffff,
379 0x8b10, 0x0000ff0f, 0x00000000,
380 0x28a4c, 0x07ffffff, 0x4e000000,
381 0x28350, 0x3f3f3fff, 0x2a00126a,
382 0x30, 0x000000ff, 0x0040,
383 0x34, 0x00000040, 0x00004040,
384 0x9100, 0x07ffffff, 0x03000000,
385 0x9060, 0x0000007f, 0x00000020,
386 0x9508, 0x00010000, 0x00010000,
387 0xac14, 0x000003ff, 0x000000f7,
388 0xac10, 0xffffffff, 0x00000000,
389 0xac0c, 0xffffffff, 0x32761054,
390 0x88d4, 0x0000001f, 0x00000010,
391 0x15c0, 0x000c0fc0, 0x000c0400
392};
393
394static const u32 verde_golden_rlc_registers[] =
395{
396 0xc424, 0xffffffff, 0x033f1005,
397 0xc47c, 0xffffffff, 0x10808020,
398 0xc488, 0xffffffff, 0x00800008,
399 0xc314, 0xffffffff, 0x00001000,
400 0xc30c, 0xffffffff, 0x80010014
401};
402
403static const u32 verde_golden_registers[] =
404{
405 0x9a10, 0x00010000, 0x00018208,
406 0x9830, 0xffffffff, 0x00000000,
407 0x9834, 0xf00fffff, 0x00000400,
408 0x9838, 0x0002021c, 0x00020200,
409 0xc78, 0x00000080, 0x00000000,
410 0xd030, 0x000300c0, 0x00800040,
411 0xd030, 0x000300c0, 0x00800040,
412 0xd830, 0x000300c0, 0x00800040,
413 0xd830, 0x000300c0, 0x00800040,
414 0x5bb0, 0x000000f0, 0x00000070,
415 0x5bc0, 0x00200000, 0x50100000,
416 0x7030, 0x31000311, 0x00000011,
417 0x2ae4, 0x00073ffe, 0x000022a2,
418 0x2ae4, 0x00073ffe, 0x000022a2,
419 0x2ae4, 0x00073ffe, 0x000022a2,
420 0x240c, 0x000007ff, 0x00000000,
421 0x240c, 0x000007ff, 0x00000000,
422 0x240c, 0x000007ff, 0x00000000,
423 0x8a14, 0xf000001f, 0x00000007,
424 0x8a14, 0xf000001f, 0x00000007,
425 0x8a14, 0xf000001f, 0x00000007,
426 0x8b24, 0xffffffff, 0x00ffffff,
427 0x8b10, 0x0000ff0f, 0x00000000,
428 0x28a4c, 0x07ffffff, 0x4e000000,
429 0x28350, 0x3f3f3fff, 0x0000124a,
430 0x28350, 0x3f3f3fff, 0x0000124a,
431 0x28350, 0x3f3f3fff, 0x0000124a,
432 0x30, 0x000000ff, 0x0040,
433 0x34, 0x00000040, 0x00004040,
434 0x9100, 0x07ffffff, 0x03000000,
435 0x9100, 0x07ffffff, 0x03000000,
436 0x8e88, 0x01ff1f3f, 0x00000000,
437 0x8e88, 0x01ff1f3f, 0x00000000,
438 0x8e88, 0x01ff1f3f, 0x00000000,
439 0x8e84, 0x01ff1f3f, 0x00000000,
440 0x8e84, 0x01ff1f3f, 0x00000000,
441 0x8e84, 0x01ff1f3f, 0x00000000,
442 0x9060, 0x0000007f, 0x00000020,
443 0x9508, 0x00010000, 0x00010000,
444 0xac14, 0x000003ff, 0x00000003,
445 0xac14, 0x000003ff, 0x00000003,
446 0xac14, 0x000003ff, 0x00000003,
447 0xac10, 0xffffffff, 0x00000000,
448 0xac10, 0xffffffff, 0x00000000,
449 0xac10, 0xffffffff, 0x00000000,
450 0xac0c, 0xffffffff, 0x00001032,
451 0xac0c, 0xffffffff, 0x00001032,
452 0xac0c, 0xffffffff, 0x00001032,
453 0x88d4, 0x0000001f, 0x00000010,
454 0x88d4, 0x0000001f, 0x00000010,
455 0x88d4, 0x0000001f, 0x00000010,
456 0x15c0, 0x000c0fc0, 0x000c0400
457};
458
459static const u32 oland_golden_rlc_registers[] =
460{
461 0xc424, 0xffffffff, 0x00601005,
462 0xc47c, 0xffffffff, 0x10104040,
463 0xc488, 0xffffffff, 0x0100000a,
464 0xc314, 0xffffffff, 0x00000800,
465 0xc30c, 0xffffffff, 0x800000f4
466};
467
468static const u32 oland_golden_registers[] =
469{
470 0x9a10, 0x00010000, 0x00018208,
471 0x9830, 0xffffffff, 0x00000000,
472 0x9834, 0xf00fffff, 0x00000400,
473 0x9838, 0x0002021c, 0x00020200,
474 0xc78, 0x00000080, 0x00000000,
475 0xd030, 0x000300c0, 0x00800040,
476 0xd830, 0x000300c0, 0x00800040,
477 0x5bb0, 0x000000f0, 0x00000070,
478 0x5bc0, 0x00200000, 0x50100000,
479 0x7030, 0x31000311, 0x00000011,
480 0x2ae4, 0x00073ffe, 0x000022a2,
481 0x240c, 0x000007ff, 0x00000000,
482 0x8a14, 0xf000001f, 0x00000007,
483 0x8b24, 0xffffffff, 0x00ffffff,
484 0x8b10, 0x0000ff0f, 0x00000000,
485 0x28a4c, 0x07ffffff, 0x4e000000,
486 0x28350, 0x3f3f3fff, 0x00000082,
487 0x30, 0x000000ff, 0x0040,
488 0x34, 0x00000040, 0x00004040,
489 0x9100, 0x07ffffff, 0x03000000,
490 0x9060, 0x0000007f, 0x00000020,
491 0x9508, 0x00010000, 0x00010000,
492 0xac14, 0x000003ff, 0x000000f3,
493 0xac10, 0xffffffff, 0x00000000,
494 0xac0c, 0xffffffff, 0x00003210,
495 0x88d4, 0x0000001f, 0x00000010,
496 0x15c0, 0x000c0fc0, 0x000c0400
497};
498
fffbdda4
AD		 499static const u32 hainan_golden_registers[] =
500{
501 0x9a10, 0x00010000, 0x00018208,
502 0x9830, 0xffffffff, 0x00000000,
503 0x9834, 0xf00fffff, 0x00000400,
504 0x9838, 0x0002021c, 0x00020200,
505 0xd0c0, 0xff000fff, 0x00000100,
506 0xd030, 0x000300c0, 0x00800040,
507 0xd8c0, 0xff000fff, 0x00000100,
508 0xd830, 0x000300c0, 0x00800040,
509 0x2ae4, 0x00073ffe, 0x000022a2,
510 0x240c, 0x000007ff, 0x00000000,
511 0x8a14, 0xf000001f, 0x00000007,
512 0x8b24, 0xffffffff, 0x00ffffff,
513 0x8b10, 0x0000ff0f, 0x00000000,
514 0x28a4c, 0x07ffffff, 0x4e000000,
515 0x28350, 0x3f3f3fff, 0x00000000,
516 0x30, 0x000000ff, 0x0040,
517 0x34, 0x00000040, 0x00004040,
518 0x9100, 0x03e00000, 0x03600000,
519 0x9060, 0x0000007f, 0x00000020,
520 0x9508, 0x00010000, 0x00010000,
521 0xac14, 0x000003ff, 0x000000f1,
522 0xac10, 0xffffffff, 0x00000000,
523 0xac0c, 0xffffffff, 0x00003210,
524 0x88d4, 0x0000001f, 0x00000010,
525 0x15c0, 0x000c0fc0, 0x000c0400
526};
527
528static const u32 hainan_golden_registers2[] =
529{
530 0x98f8, 0xffffffff, 0x02010001
531};
532
205996c0
AD		 533static const u32 tahiti_mgcg_cgcg_init[] =
534{
535 0xc400, 0xffffffff, 0xfffffffc,
536 0x802c, 0xffffffff, 0xe0000000,
537 0x9a60, 0xffffffff, 0x00000100,
538 0x92a4, 0xffffffff, 0x00000100,
539 0xc164, 0xffffffff, 0x00000100,
540 0x9774, 0xffffffff, 0x00000100,
541 0x8984, 0xffffffff, 0x06000100,
542 0x8a18, 0xffffffff, 0x00000100,
543 0x92a0, 0xffffffff, 0x00000100,
544 0xc380, 0xffffffff, 0x00000100,
545 0x8b28, 0xffffffff, 0x00000100,
546 0x9144, 0xffffffff, 0x00000100,
547 0x8d88, 0xffffffff, 0x00000100,
548 0x8d8c, 0xffffffff, 0x00000100,
549 0x9030, 0xffffffff, 0x00000100,
550 0x9034, 0xffffffff, 0x00000100,
551 0x9038, 0xffffffff, 0x00000100,
552 0x903c, 0xffffffff, 0x00000100,
553 0xad80, 0xffffffff, 0x00000100,
554 0xac54, 0xffffffff, 0x00000100,
555 0x897c, 0xffffffff, 0x06000100,
556 0x9868, 0xffffffff, 0x00000100,
557 0x9510, 0xffffffff, 0x00000100,
558 0xaf04, 0xffffffff, 0x00000100,
559 0xae04, 0xffffffff, 0x00000100,
560 0x949c, 0xffffffff, 0x00000100,
561 0x802c, 0xffffffff, 0xe0000000,
562 0x9160, 0xffffffff, 0x00010000,
563 0x9164, 0xffffffff, 0x00030002,
564 0x9168, 0xffffffff, 0x00040007,
565 0x916c, 0xffffffff, 0x00060005,
566 0x9170, 0xffffffff, 0x00090008,
567 0x9174, 0xffffffff, 0x00020001,
568 0x9178, 0xffffffff, 0x00040003,
569 0x917c, 0xffffffff, 0x00000007,
570 0x9180, 0xffffffff, 0x00060005,
571 0x9184, 0xffffffff, 0x00090008,
572 0x9188, 0xffffffff, 0x00030002,
573 0x918c, 0xffffffff, 0x00050004,
574 0x9190, 0xffffffff, 0x00000008,
575 0x9194, 0xffffffff, 0x00070006,
576 0x9198, 0xffffffff, 0x000a0009,
577 0x919c, 0xffffffff, 0x00040003,
578 0x91a0, 0xffffffff, 0x00060005,
579 0x91a4, 0xffffffff, 0x00000009,
580 0x91a8, 0xffffffff, 0x00080007,
581 0x91ac, 0xffffffff, 0x000b000a,
582 0x91b0, 0xffffffff, 0x00050004,
583 0x91b4, 0xffffffff, 0x00070006,
584 0x91b8, 0xffffffff, 0x0008000b,
585 0x91bc, 0xffffffff, 0x000a0009,
586 0x91c0, 0xffffffff, 0x000d000c,
587 0x91c4, 0xffffffff, 0x00060005,
588 0x91c8, 0xffffffff, 0x00080007,
589 0x91cc, 0xffffffff, 0x0000000b,
590 0x91d0, 0xffffffff, 0x000a0009,
591 0x91d4, 0xffffffff, 0x000d000c,
592 0x91d8, 0xffffffff, 0x00070006,
593 0x91dc, 0xffffffff, 0x00090008,
594 0x91e0, 0xffffffff, 0x0000000c,
595 0x91e4, 0xffffffff, 0x000b000a,
596 0x91e8, 0xffffffff, 0x000e000d,
597 0x91ec, 0xffffffff, 0x00080007,
598 0x91f0, 0xffffffff, 0x000a0009,
599 0x91f4, 0xffffffff, 0x0000000d,
600 0x91f8, 0xffffffff, 0x000c000b,
601 0x91fc, 0xffffffff, 0x000f000e,
602 0x9200, 0xffffffff, 0x00090008,
603 0x9204, 0xffffffff, 0x000b000a,
604 0x9208, 0xffffffff, 0x000c000f,
605 0x920c, 0xffffffff, 0x000e000d,
606 0x9210, 0xffffffff, 0x00110010,
607 0x9214, 0xffffffff, 0x000a0009,
608 0x9218, 0xffffffff, 0x000c000b,
609 0x921c, 0xffffffff, 0x0000000f,
610 0x9220, 0xffffffff, 0x000e000d,
611 0x9224, 0xffffffff, 0x00110010,
612 0x9228, 0xffffffff, 0x000b000a,
613 0x922c, 0xffffffff, 0x000d000c,
614 0x9230, 0xffffffff, 0x00000010,
615 0x9234, 0xffffffff, 0x000f000e,
616 0x9238, 0xffffffff, 0x00120011,
617 0x923c, 0xffffffff, 0x000c000b,
618 0x9240, 0xffffffff, 0x000e000d,
619 0x9244, 0xffffffff, 0x00000011,
620 0x9248, 0xffffffff, 0x0010000f,
621 0x924c, 0xffffffff, 0x00130012,
622 0x9250, 0xffffffff, 0x000d000c,
623 0x9254, 0xffffffff, 0x000f000e,
624 0x9258, 0xffffffff, 0x00100013,
625 0x925c, 0xffffffff, 0x00120011,
626 0x9260, 0xffffffff, 0x00150014,
627 0x9264, 0xffffffff, 0x000e000d,
628 0x9268, 0xffffffff, 0x0010000f,
629 0x926c, 0xffffffff, 0x00000013,
630 0x9270, 0xffffffff, 0x00120011,
631 0x9274, 0xffffffff, 0x00150014,
632 0x9278, 0xffffffff, 0x000f000e,
633 0x927c, 0xffffffff, 0x00110010,
634 0x9280, 0xffffffff, 0x00000014,
635 0x9284, 0xffffffff, 0x00130012,
636 0x9288, 0xffffffff, 0x00160015,
637 0x928c, 0xffffffff, 0x0010000f,
638 0x9290, 0xffffffff, 0x00120011,
639 0x9294, 0xffffffff, 0x00000015,
640 0x9298, 0xffffffff, 0x00140013,
641 0x929c, 0xffffffff, 0x00170016,
642 0x9150, 0xffffffff, 0x96940200,
643 0x8708, 0xffffffff, 0x00900100,
644 0xc478, 0xffffffff, 0x00000080,
645 0xc404, 0xffffffff, 0x0020003f,
646 0x30, 0xffffffff, 0x0000001c,
647 0x34, 0x000f0000, 0x000f0000,
648 0x160c, 0xffffffff, 0x00000100,
649 0x1024, 0xffffffff, 0x00000100,
650 0x102c, 0x00000101, 0x00000000,
651 0x20a8, 0xffffffff, 0x00000104,
652 0x264c, 0x000c0000, 0x000c0000,
653 0x2648, 0x000c0000, 0x000c0000,
654 0x55e4, 0xff000fff, 0x00000100,
655 0x55e8, 0x00000001, 0x00000001,
656 0x2f50, 0x00000001, 0x00000001,
657 0x30cc, 0xc0000fff, 0x00000104,
658 0xc1e4, 0x00000001, 0x00000001,
659 0xd0c0, 0xfffffff0, 0x00000100,
660 0xd8c0, 0xfffffff0, 0x00000100
661};
662
663static const u32 pitcairn_mgcg_cgcg_init[] =
664{
665 0xc400, 0xffffffff, 0xfffffffc,
666 0x802c, 0xffffffff, 0xe0000000,
667 0x9a60, 0xffffffff, 0x00000100,
668 0x92a4, 0xffffffff, 0x00000100,
669 0xc164, 0xffffffff, 0x00000100,
670 0x9774, 0xffffffff, 0x00000100,
671 0x8984, 0xffffffff, 0x06000100,
672 0x8a18, 0xffffffff, 0x00000100,
673 0x92a0, 0xffffffff, 0x00000100,
674 0xc380, 0xffffffff, 0x00000100,
675 0x8b28, 0xffffffff, 0x00000100,
676 0x9144, 0xffffffff, 0x00000100,
677 0x8d88, 0xffffffff, 0x00000100,
678 0x8d8c, 0xffffffff, 0x00000100,
679 0x9030, 0xffffffff, 0x00000100,
680 0x9034, 0xffffffff, 0x00000100,
681 0x9038, 0xffffffff, 0x00000100,
682 0x903c, 0xffffffff, 0x00000100,
683 0xad80, 0xffffffff, 0x00000100,
684 0xac54, 0xffffffff, 0x00000100,
685 0x897c, 0xffffffff, 0x06000100,
686 0x9868, 0xffffffff, 0x00000100,
687 0x9510, 0xffffffff, 0x00000100,
688 0xaf04, 0xffffffff, 0x00000100,
689 0xae04, 0xffffffff, 0x00000100,
690 0x949c, 0xffffffff, 0x00000100,
691 0x802c, 0xffffffff, 0xe0000000,
692 0x9160, 0xffffffff, 0x00010000,
693 0x9164, 0xffffffff, 0x00030002,
694 0x9168, 0xffffffff, 0x00040007,
695 0x916c, 0xffffffff, 0x00060005,
696 0x9170, 0xffffffff, 0x00090008,
697 0x9174, 0xffffffff, 0x00020001,
698 0x9178, 0xffffffff, 0x00040003,
699 0x917c, 0xffffffff, 0x00000007,
700 0x9180, 0xffffffff, 0x00060005,
701 0x9184, 0xffffffff, 0x00090008,
702 0x9188, 0xffffffff, 0x00030002,
703 0x918c, 0xffffffff, 0x00050004,
704 0x9190, 0xffffffff, 0x00000008,
705 0x9194, 0xffffffff, 0x00070006,
706 0x9198, 0xffffffff, 0x000a0009,
707 0x919c, 0xffffffff, 0x00040003,
708 0x91a0, 0xffffffff, 0x00060005,
709 0x91a4, 0xffffffff, 0x00000009,
710 0x91a8, 0xffffffff, 0x00080007,
711 0x91ac, 0xffffffff, 0x000b000a,
712 0x91b0, 0xffffffff, 0x00050004,
713 0x91b4, 0xffffffff, 0x00070006,
714 0x91b8, 0xffffffff, 0x0008000b,
715 0x91bc, 0xffffffff, 0x000a0009,
716 0x91c0, 0xffffffff, 0x000d000c,
717 0x9200, 0xffffffff, 0x00090008,
718 0x9204, 0xffffffff, 0x000b000a,
719 0x9208, 0xffffffff, 0x000c000f,
720 0x920c, 0xffffffff, 0x000e000d,
721 0x9210, 0xffffffff, 0x00110010,
722 0x9214, 0xffffffff, 0x000a0009,
723 0x9218, 0xffffffff, 0x000c000b,
724 0x921c, 0xffffffff, 0x0000000f,
725 0x9220, 0xffffffff, 0x000e000d,
726 0x9224, 0xffffffff, 0x00110010,
727 0x9228, 0xffffffff, 0x000b000a,
728 0x922c, 0xffffffff, 0x000d000c,
729 0x9230, 0xffffffff, 0x00000010,
730 0x9234, 0xffffffff, 0x000f000e,
731 0x9238, 0xffffffff, 0x00120011,
732 0x923c, 0xffffffff, 0x000c000b,
733 0x9240, 0xffffffff, 0x000e000d,
734 0x9244, 0xffffffff, 0x00000011,
735 0x9248, 0xffffffff, 0x0010000f,
736 0x924c, 0xffffffff, 0x00130012,
737 0x9250, 0xffffffff, 0x000d000c,
738 0x9254, 0xffffffff, 0x000f000e,
739 0x9258, 0xffffffff, 0x00100013,
740 0x925c, 0xffffffff, 0x00120011,
741 0x9260, 0xffffffff, 0x00150014,
742 0x9150, 0xffffffff, 0x96940200,
743 0x8708, 0xffffffff, 0x00900100,
744 0xc478, 0xffffffff, 0x00000080,
745 0xc404, 0xffffffff, 0x0020003f,
746 0x30, 0xffffffff, 0x0000001c,
747 0x34, 0x000f0000, 0x000f0000,
748 0x160c, 0xffffffff, 0x00000100,
749 0x1024, 0xffffffff, 0x00000100,
750 0x102c, 0x00000101, 0x00000000,
751 0x20a8, 0xffffffff, 0x00000104,
752 0x55e4, 0xff000fff, 0x00000100,
753 0x55e8, 0x00000001, 0x00000001,
754 0x2f50, 0x00000001, 0x00000001,
755 0x30cc, 0xc0000fff, 0x00000104,
756 0xc1e4, 0x00000001, 0x00000001,
757 0xd0c0, 0xfffffff0, 0x00000100,
758 0xd8c0, 0xfffffff0, 0x00000100
759};
760
761static const u32 verde_mgcg_cgcg_init[] =
762{
763 0xc400, 0xffffffff, 0xfffffffc,
764 0x802c, 0xffffffff, 0xe0000000,
765 0x9a60, 0xffffffff, 0x00000100,
766 0x92a4, 0xffffffff, 0x00000100,
767 0xc164, 0xffffffff, 0x00000100,
768 0x9774, 0xffffffff, 0x00000100,
769 0x8984, 0xffffffff, 0x06000100,
770 0x8a18, 0xffffffff, 0x00000100,
771 0x92a0, 0xffffffff, 0x00000100,
772 0xc380, 0xffffffff, 0x00000100,
773 0x8b28, 0xffffffff, 0x00000100,
774 0x9144, 0xffffffff, 0x00000100,
775 0x8d88, 0xffffffff, 0x00000100,
776 0x8d8c, 0xffffffff, 0x00000100,
777 0x9030, 0xffffffff, 0x00000100,
778 0x9034, 0xffffffff, 0x00000100,
779 0x9038, 0xffffffff, 0x00000100,
780 0x903c, 0xffffffff, 0x00000100,
781 0xad80, 0xffffffff, 0x00000100,
782 0xac54, 0xffffffff, 0x00000100,
783 0x897c, 0xffffffff, 0x06000100,
784 0x9868, 0xffffffff, 0x00000100,
785 0x9510, 0xffffffff, 0x00000100,
786 0xaf04, 0xffffffff, 0x00000100,
787 0xae04, 0xffffffff, 0x00000100,
788 0x949c, 0xffffffff, 0x00000100,
789 0x802c, 0xffffffff, 0xe0000000,
790 0x9160, 0xffffffff, 0x00010000,
791 0x9164, 0xffffffff, 0x00030002,
792 0x9168, 0xffffffff, 0x00040007,
793 0x916c, 0xffffffff, 0x00060005,
794 0x9170, 0xffffffff, 0x00090008,
795 0x9174, 0xffffffff, 0x00020001,
796 0x9178, 0xffffffff, 0x00040003,
797 0x917c, 0xffffffff, 0x00000007,
798 0x9180, 0xffffffff, 0x00060005,
799 0x9184, 0xffffffff, 0x00090008,
800 0x9188, 0xffffffff, 0x00030002,
801 0x918c, 0xffffffff, 0x00050004,
802 0x9190, 0xffffffff, 0x00000008,
803 0x9194, 0xffffffff, 0x00070006,
804 0x9198, 0xffffffff, 0x000a0009,
805 0x919c, 0xffffffff, 0x00040003,
806 0x91a0, 0xffffffff, 0x00060005,
807 0x91a4, 0xffffffff, 0x00000009,
808 0x91a8, 0xffffffff, 0x00080007,
809 0x91ac, 0xffffffff, 0x000b000a,
810 0x91b0, 0xffffffff, 0x00050004,
811 0x91b4, 0xffffffff, 0x00070006,
812 0x91b8, 0xffffffff, 0x0008000b,
813 0x91bc, 0xffffffff, 0x000a0009,
814 0x91c0, 0xffffffff, 0x000d000c,
815 0x9200, 0xffffffff, 0x00090008,
816 0x9204, 0xffffffff, 0x000b000a,
817 0x9208, 0xffffffff, 0x000c000f,
818 0x920c, 0xffffffff, 0x000e000d,
819 0x9210, 0xffffffff, 0x00110010,
820 0x9214, 0xffffffff, 0x000a0009,
821 0x9218, 0xffffffff, 0x000c000b,
822 0x921c, 0xffffffff, 0x0000000f,
823 0x9220, 0xffffffff, 0x000e000d,
824 0x9224, 0xffffffff, 0x00110010,
825 0x9228, 0xffffffff, 0x000b000a,
826 0x922c, 0xffffffff, 0x000d000c,
827 0x9230, 0xffffffff, 0x00000010,
828 0x9234, 0xffffffff, 0x000f000e,
829 0x9238, 0xffffffff, 0x00120011,
830 0x923c, 0xffffffff, 0x000c000b,
831 0x9240, 0xffffffff, 0x000e000d,
832 0x9244, 0xffffffff, 0x00000011,
833 0x9248, 0xffffffff, 0x0010000f,
834 0x924c, 0xffffffff, 0x00130012,
835 0x9250, 0xffffffff, 0x000d000c,
836 0x9254, 0xffffffff, 0x000f000e,
837 0x9258, 0xffffffff, 0x00100013,
838 0x925c, 0xffffffff, 0x00120011,
839 0x9260, 0xffffffff, 0x00150014,
840 0x9150, 0xffffffff, 0x96940200,
841 0x8708, 0xffffffff, 0x00900100,
842 0xc478, 0xffffffff, 0x00000080,
843 0xc404, 0xffffffff, 0x0020003f,
844 0x30, 0xffffffff, 0x0000001c,
845 0x34, 0x000f0000, 0x000f0000,
846 0x160c, 0xffffffff, 0x00000100,
847 0x1024, 0xffffffff, 0x00000100,
848 0x102c, 0x00000101, 0x00000000,
849 0x20a8, 0xffffffff, 0x00000104,
850 0x264c, 0x000c0000, 0x000c0000,
851 0x2648, 0x000c0000, 0x000c0000,
852 0x55e4, 0xff000fff, 0x00000100,
853 0x55e8, 0x00000001, 0x00000001,
854 0x2f50, 0x00000001, 0x00000001,
855 0x30cc, 0xc0000fff, 0x00000104,
856 0xc1e4, 0x00000001, 0x00000001,
857 0xd0c0, 0xfffffff0, 0x00000100,
858 0xd8c0, 0xfffffff0, 0x00000100
859};
860
861static const u32 oland_mgcg_cgcg_init[] =
862{
863 0xc400, 0xffffffff, 0xfffffffc,
864 0x802c, 0xffffffff, 0xe0000000,
865 0x9a60, 0xffffffff, 0x00000100,
866 0x92a4, 0xffffffff, 0x00000100,
867 0xc164, 0xffffffff, 0x00000100,
868 0x9774, 0xffffffff, 0x00000100,
869 0x8984, 0xffffffff, 0x06000100,
870 0x8a18, 0xffffffff, 0x00000100,
871 0x92a0, 0xffffffff, 0x00000100,
872 0xc380, 0xffffffff, 0x00000100,
873 0x8b28, 0xffffffff, 0x00000100,
874 0x9144, 0xffffffff, 0x00000100,
875 0x8d88, 0xffffffff, 0x00000100,
876 0x8d8c, 0xffffffff, 0x00000100,
877 0x9030, 0xffffffff, 0x00000100,
878 0x9034, 0xffffffff, 0x00000100,
879 0x9038, 0xffffffff, 0x00000100,
880 0x903c, 0xffffffff, 0x00000100,
881 0xad80, 0xffffffff, 0x00000100,
882 0xac54, 0xffffffff, 0x00000100,
883 0x897c, 0xffffffff, 0x06000100,
884 0x9868, 0xffffffff, 0x00000100,
885 0x9510, 0xffffffff, 0x00000100,
886 0xaf04, 0xffffffff, 0x00000100,
887 0xae04, 0xffffffff, 0x00000100,
888 0x949c, 0xffffffff, 0x00000100,
889 0x802c, 0xffffffff, 0xe0000000,
890 0x9160, 0xffffffff, 0x00010000,
891 0x9164, 0xffffffff, 0x00030002,
892 0x9168, 0xffffffff, 0x00040007,
893 0x916c, 0xffffffff, 0x00060005,
894 0x9170, 0xffffffff, 0x00090008,
895 0x9174, 0xffffffff, 0x00020001,
896 0x9178, 0xffffffff, 0x00040003,
897 0x917c, 0xffffffff, 0x00000007,
898 0x9180, 0xffffffff, 0x00060005,
899 0x9184, 0xffffffff, 0x00090008,
900 0x9188, 0xffffffff, 0x00030002,
901 0x918c, 0xffffffff, 0x00050004,
902 0x9190, 0xffffffff, 0x00000008,
903 0x9194, 0xffffffff, 0x00070006,
904 0x9198, 0xffffffff, 0x000a0009,
905 0x919c, 0xffffffff, 0x00040003,
906 0x91a0, 0xffffffff, 0x00060005,
907 0x91a4, 0xffffffff, 0x00000009,
908 0x91a8, 0xffffffff, 0x00080007,
909 0x91ac, 0xffffffff, 0x000b000a,
910 0x91b0, 0xffffffff, 0x00050004,
911 0x91b4, 0xffffffff, 0x00070006,
912 0x91b8, 0xffffffff, 0x0008000b,
913 0x91bc, 0xffffffff, 0x000a0009,
914 0x91c0, 0xffffffff, 0x000d000c,
915 0x91c4, 0xffffffff, 0x00060005,
916 0x91c8, 0xffffffff, 0x00080007,
917 0x91cc, 0xffffffff, 0x0000000b,
918 0x91d0, 0xffffffff, 0x000a0009,
919 0x91d4, 0xffffffff, 0x000d000c,
920 0x9150, 0xffffffff, 0x96940200,
921 0x8708, 0xffffffff, 0x00900100,
922 0xc478, 0xffffffff, 0x00000080,
923 0xc404, 0xffffffff, 0x0020003f,
924 0x30, 0xffffffff, 0x0000001c,
925 0x34, 0x000f0000, 0x000f0000,
926 0x160c, 0xffffffff, 0x00000100,
927 0x1024, 0xffffffff, 0x00000100,
928 0x102c, 0x00000101, 0x00000000,
929 0x20a8, 0xffffffff, 0x00000104,
930 0x264c, 0x000c0000, 0x000c0000,
931 0x2648, 0x000c0000, 0x000c0000,
932 0x55e4, 0xff000fff, 0x00000100,
933 0x55e8, 0x00000001, 0x00000001,
934 0x2f50, 0x00000001, 0x00000001,
935 0x30cc, 0xc0000fff, 0x00000104,
936 0xc1e4, 0x00000001, 0x00000001,
937 0xd0c0, 0xfffffff0, 0x00000100,
938 0xd8c0, 0xfffffff0, 0x00000100
939};
940
fffbdda4
AD		 941static const u32 hainan_mgcg_cgcg_init[] =
942{
943 0xc400, 0xffffffff, 0xfffffffc,
944 0x802c, 0xffffffff, 0xe0000000,
945 0x9a60, 0xffffffff, 0x00000100,
946 0x92a4, 0xffffffff, 0x00000100,
947 0xc164, 0xffffffff, 0x00000100,
948 0x9774, 0xffffffff, 0x00000100,
949 0x8984, 0xffffffff, 0x06000100,
950 0x8a18, 0xffffffff, 0x00000100,
951 0x92a0, 0xffffffff, 0x00000100,
952 0xc380, 0xffffffff, 0x00000100,
953 0x8b28, 0xffffffff, 0x00000100,
954 0x9144, 0xffffffff, 0x00000100,
955 0x8d88, 0xffffffff, 0x00000100,
956 0x8d8c, 0xffffffff, 0x00000100,
957 0x9030, 0xffffffff, 0x00000100,
958 0x9034, 0xffffffff, 0x00000100,
959 0x9038, 0xffffffff, 0x00000100,
960 0x903c, 0xffffffff, 0x00000100,
961 0xad80, 0xffffffff, 0x00000100,
962 0xac54, 0xffffffff, 0x00000100,
963 0x897c, 0xffffffff, 0x06000100,
964 0x9868, 0xffffffff, 0x00000100,
965 0x9510, 0xffffffff, 0x00000100,
966 0xaf04, 0xffffffff, 0x00000100,
967 0xae04, 0xffffffff, 0x00000100,
968 0x949c, 0xffffffff, 0x00000100,
969 0x802c, 0xffffffff, 0xe0000000,
970 0x9160, 0xffffffff, 0x00010000,
971 0x9164, 0xffffffff, 0x00030002,
972 0x9168, 0xffffffff, 0x00040007,
973 0x916c, 0xffffffff, 0x00060005,
974 0x9170, 0xffffffff, 0x00090008,
975 0x9174, 0xffffffff, 0x00020001,
976 0x9178, 0xffffffff, 0x00040003,
977 0x917c, 0xffffffff, 0x00000007,
978 0x9180, 0xffffffff, 0x00060005,
979 0x9184, 0xffffffff, 0x00090008,
980 0x9188, 0xffffffff, 0x00030002,
981 0x918c, 0xffffffff, 0x00050004,
982 0x9190, 0xffffffff, 0x00000008,
983 0x9194, 0xffffffff, 0x00070006,
984 0x9198, 0xffffffff, 0x000a0009,
985 0x919c, 0xffffffff, 0x00040003,
986 0x91a0, 0xffffffff, 0x00060005,
987 0x91a4, 0xffffffff, 0x00000009,
988 0x91a8, 0xffffffff, 0x00080007,
989 0x91ac, 0xffffffff, 0x000b000a,
990 0x91b0, 0xffffffff, 0x00050004,
991 0x91b4, 0xffffffff, 0x00070006,
992 0x91b8, 0xffffffff, 0x0008000b,
993 0x91bc, 0xffffffff, 0x000a0009,
994 0x91c0, 0xffffffff, 0x000d000c,
995 0x91c4, 0xffffffff, 0x00060005,
996 0x91c8, 0xffffffff, 0x00080007,
997 0x91cc, 0xffffffff, 0x0000000b,
998 0x91d0, 0xffffffff, 0x000a0009,
999 0x91d4, 0xffffffff, 0x000d000c,
1000 0x9150, 0xffffffff, 0x96940200,
1001 0x8708, 0xffffffff, 0x00900100,
1002 0xc478, 0xffffffff, 0x00000080,
1003 0xc404, 0xffffffff, 0x0020003f,
1004 0x30, 0xffffffff, 0x0000001c,
1005 0x34, 0x000f0000, 0x000f0000,
1006 0x160c, 0xffffffff, 0x00000100,
1007 0x1024, 0xffffffff, 0x00000100,
1008 0x20a8, 0xffffffff, 0x00000104,
1009 0x264c, 0x000c0000, 0x000c0000,
1010 0x2648, 0x000c0000, 0x000c0000,
1011 0x2f50, 0x00000001, 0x00000001,
1012 0x30cc, 0xc0000fff, 0x00000104,
1013 0xc1e4, 0x00000001, 0x00000001,
1014 0xd0c0, 0xfffffff0, 0x00000100,
1015 0xd8c0, 0xfffffff0, 0x00000100
1016};
1017
205996c0
AD		 1018static u32 verde_pg_init[] =
1019{
1020 0x353c, 0xffffffff, 0x40000,
1021 0x3538, 0xffffffff, 0x200010ff,
1022 0x353c, 0xffffffff, 0x0,
1023 0x353c, 0xffffffff, 0x0,
1024 0x353c, 0xffffffff, 0x0,
1025 0x353c, 0xffffffff, 0x0,
1026 0x353c, 0xffffffff, 0x0,
1027 0x353c, 0xffffffff, 0x7007,
1028 0x3538, 0xffffffff, 0x300010ff,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x400000,
1035 0x3538, 0xffffffff, 0x100010ff,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x120200,
1042 0x3538, 0xffffffff, 0x500010ff,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x1e1e16,
1049 0x3538, 0xffffffff, 0x600010ff,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x171f1e,
1056 0x3538, 0xffffffff, 0x700010ff,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x353c, 0xffffffff, 0x0,
1063 0x3538, 0xffffffff, 0x9ff,
1064 0x3500, 0xffffffff, 0x0,
1065 0x3504, 0xffffffff, 0x10000800,
1066 0x3504, 0xffffffff, 0xf,
1067 0x3504, 0xffffffff, 0xf,
1068 0x3500, 0xffffffff, 0x4,
1069 0x3504, 0xffffffff, 0x1000051e,
1070 0x3504, 0xffffffff, 0xffff,
1071 0x3504, 0xffffffff, 0xffff,
1072 0x3500, 0xffffffff, 0x8,
1073 0x3504, 0xffffffff, 0x80500,
1074 0x3500, 0xffffffff, 0x12,
1075 0x3504, 0xffffffff, 0x9050c,
1076 0x3500, 0xffffffff, 0x1d,
1077 0x3504, 0xffffffff, 0xb052c,
1078 0x3500, 0xffffffff, 0x2a,
1079 0x3504, 0xffffffff, 0x1053e,
1080 0x3500, 0xffffffff, 0x2d,
1081 0x3504, 0xffffffff, 0x10546,
1082 0x3500, 0xffffffff, 0x30,
1083 0x3504, 0xffffffff, 0xa054e,
1084 0x3500, 0xffffffff, 0x3c,
1085 0x3504, 0xffffffff, 0x1055f,
1086 0x3500, 0xffffffff, 0x3f,
1087 0x3504, 0xffffffff, 0x10567,
1088 0x3500, 0xffffffff, 0x42,
1089 0x3504, 0xffffffff, 0x1056f,
1090 0x3500, 0xffffffff, 0x45,
1091 0x3504, 0xffffffff, 0x10572,
1092 0x3500, 0xffffffff, 0x48,
1093 0x3504, 0xffffffff, 0x20575,
1094 0x3500, 0xffffffff, 0x4c,
1095 0x3504, 0xffffffff, 0x190801,
1096 0x3500, 0xffffffff, 0x67,
1097 0x3504, 0xffffffff, 0x1082a,
1098 0x3500, 0xffffffff, 0x6a,
1099 0x3504, 0xffffffff, 0x1b082d,
1100 0x3500, 0xffffffff, 0x87,
1101 0x3504, 0xffffffff, 0x310851,
1102 0x3500, 0xffffffff, 0xba,
1103 0x3504, 0xffffffff, 0x891,
1104 0x3500, 0xffffffff, 0xbc,
1105 0x3504, 0xffffffff, 0x893,
1106 0x3500, 0xffffffff, 0xbe,
1107 0x3504, 0xffffffff, 0x20895,
1108 0x3500, 0xffffffff, 0xc2,
1109 0x3504, 0xffffffff, 0x20899,
1110 0x3500, 0xffffffff, 0xc6,
1111 0x3504, 0xffffffff, 0x2089d,
1112 0x3500, 0xffffffff, 0xca,
1113 0x3504, 0xffffffff, 0x8a1,
1114 0x3500, 0xffffffff, 0xcc,
1115 0x3504, 0xffffffff, 0x8a3,
1116 0x3500, 0xffffffff, 0xce,
1117 0x3504, 0xffffffff, 0x308a5,
1118 0x3500, 0xffffffff, 0xd3,
1119 0x3504, 0xffffffff, 0x6d08cd,
1120 0x3500, 0xffffffff, 0x142,
1121 0x3504, 0xffffffff, 0x2000095a,
1122 0x3504, 0xffffffff, 0x1,
1123 0x3500, 0xffffffff, 0x144,
1124 0x3504, 0xffffffff, 0x301f095b,
1125 0x3500, 0xffffffff, 0x165,
1126 0x3504, 0xffffffff, 0xc094d,
1127 0x3500, 0xffffffff, 0x173,
1128 0x3504, 0xffffffff, 0xf096d,
1129 0x3500, 0xffffffff, 0x184,
1130 0x3504, 0xffffffff, 0x15097f,
1131 0x3500, 0xffffffff, 0x19b,
1132 0x3504, 0xffffffff, 0xc0998,
1133 0x3500, 0xffffffff, 0x1a9,
1134 0x3504, 0xffffffff, 0x409a7,
1135 0x3500, 0xffffffff, 0x1af,
1136 0x3504, 0xffffffff, 0xcdc,
1137 0x3500, 0xffffffff, 0x1b1,
1138 0x3504, 0xffffffff, 0x800,
1139 0x3508, 0xffffffff, 0x6c9b2000,
1140 0x3510, 0xfc00, 0x2000,
1141 0x3544, 0xffffffff, 0xfc0,
1142 0x28d4, 0x00000100, 0x100
1143};
1144
1145static void si_init_golden_registers(struct radeon_device *rdev)
1146{
1147 switch (rdev->family) {
1148 case CHIP_TAHITI:
1149 radeon_program_register_sequence(rdev,
1150 tahiti_golden_registers,
1151 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1152 radeon_program_register_sequence(rdev,
1153 tahiti_golden_rlc_registers,
1154 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1155 radeon_program_register_sequence(rdev,
1156 tahiti_mgcg_cgcg_init,
1157 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1158 radeon_program_register_sequence(rdev,
1159 tahiti_golden_registers2,
1160 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1161 break;
1162 case CHIP_PITCAIRN:
1163 radeon_program_register_sequence(rdev,
1164 pitcairn_golden_registers,
1165 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1166 radeon_program_register_sequence(rdev,
1167 pitcairn_golden_rlc_registers,
1168 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1169 radeon_program_register_sequence(rdev,
1170 pitcairn_mgcg_cgcg_init,
1171 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1172 break;
1173 case CHIP_VERDE:
1174 radeon_program_register_sequence(rdev,
1175 verde_golden_registers,
1176 (const u32)ARRAY_SIZE(verde_golden_registers));
1177 radeon_program_register_sequence(rdev,
1178 verde_golden_rlc_registers,
1179 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1180 radeon_program_register_sequence(rdev,
1181 verde_mgcg_cgcg_init,
1182 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1183 radeon_program_register_sequence(rdev,
1184 verde_pg_init,
1185 (const u32)ARRAY_SIZE(verde_pg_init));
1186 break;
1187 case CHIP_OLAND:
1188 radeon_program_register_sequence(rdev,
1189 oland_golden_registers,
1190 (const u32)ARRAY_SIZE(oland_golden_registers));
1191 radeon_program_register_sequence(rdev,
1192 oland_golden_rlc_registers,
1193 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1194 radeon_program_register_sequence(rdev,
1195 oland_mgcg_cgcg_init,
1196 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1197 break;
fffbdda4
AD		 1198 case CHIP_HAINAN:
1199 radeon_program_register_sequence(rdev,
1200 hainan_golden_registers,
1201 (const u32)ARRAY_SIZE(hainan_golden_registers));
1202 radeon_program_register_sequence(rdev,
1203 hainan_golden_registers2,
1204 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1205 radeon_program_register_sequence(rdev,
1206 hainan_mgcg_cgcg_init,
1207 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1208 break;
205996c0
AD		 1209 default:
1210 break;
1211 }
1212}
1213
454d2e2a
AD		 1214#define PCIE_BUS_CLK 10000
1215#define TCLK (PCIE_BUS_CLK / 10)
1216
1217/**
1218 * si_get_xclk - get the xclk
1219 *
1220 * @rdev: radeon_device pointer
1221 *
1222 * Returns the reference clock used by the gfx engine
1223 * (SI).
1224 */
1225u32 si_get_xclk(struct radeon_device *rdev)
1226{
1227 u32 reference_clock = rdev->clock.spll.reference_freq;
1228 u32 tmp;
1229
1230 tmp = RREG32(CG_CLKPIN_CNTL_2);
1231 if (tmp & MUX_TCLK_TO_XCLK)
1232 return TCLK;
1233
1234 tmp = RREG32(CG_CLKPIN_CNTL);
1235 if (tmp & XTALIN_DIVIDE)
1236 return reference_clock / 4;
1237
1238 return reference_clock;
1239}
1240
1bd47d2e
AD		 1241/* get temperature in millidegrees */
1242int si_get_temp(struct radeon_device *rdev)
1243{
1244 u32 temp;
1245 int actual_temp = 0;
1246
1247 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1248 CTF_TEMP_SHIFT;
1249
1250 if (temp & 0x200)
1251 actual_temp = 255;
1252 else
1253 actual_temp = temp & 0x1ff;
1254
1255 actual_temp = (actual_temp * 1000);
1256
1257 return actual_temp;
1258}
1259
8b074dd6
AD		 1260#define TAHITI_IO_MC_REGS_SIZE 36
1261
1262static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1263 {0x0000006f, 0x03044000},
1264 {0x00000070, 0x0480c018},
1265 {0x00000071, 0x00000040},
1266 {0x00000072, 0x01000000},
1267 {0x00000074, 0x000000ff},
1268 {0x00000075, 0x00143400},
1269 {0x00000076, 0x08ec0800},
1270 {0x00000077, 0x040000cc},
1271 {0x00000079, 0x00000000},
1272 {0x0000007a, 0x21000409},
1273 {0x0000007c, 0x00000000},
1274 {0x0000007d, 0xe8000000},
1275 {0x0000007e, 0x044408a8},
1276 {0x0000007f, 0x00000003},
1277 {0x00000080, 0x00000000},
1278 {0x00000081, 0x01000000},
1279 {0x00000082, 0x02000000},
1280 {0x00000083, 0x00000000},
1281 {0x00000084, 0xe3f3e4f4},
1282 {0x00000085, 0x00052024},
1283 {0x00000087, 0x00000000},
1284 {0x00000088, 0x66036603},
1285 {0x00000089, 0x01000000},
1286 {0x0000008b, 0x1c0a0000},
1287 {0x0000008c, 0xff010000},
1288 {0x0000008e, 0xffffefff},
1289 {0x0000008f, 0xfff3efff},
1290 {0x00000090, 0xfff3efbf},
1291 {0x00000094, 0x00101101},
1292 {0x00000095, 0x00000fff},
1293 {0x00000096, 0x00116fff},
1294 {0x00000097, 0x60010000},
1295 {0x00000098, 0x10010000},
1296 {0x00000099, 0x00006000},
1297 {0x0000009a, 0x00001000},
1298 {0x0000009f, 0x00a77400}
1299};
1300
1301static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1302 {0x0000006f, 0x03044000},
1303 {0x00000070, 0x0480c018},
1304 {0x00000071, 0x00000040},
1305 {0x00000072, 0x01000000},
1306 {0x00000074, 0x000000ff},
1307 {0x00000075, 0x00143400},
1308 {0x00000076, 0x08ec0800},
1309 {0x00000077, 0x040000cc},
1310 {0x00000079, 0x00000000},
1311 {0x0000007a, 0x21000409},
1312 {0x0000007c, 0x00000000},
1313 {0x0000007d, 0xe8000000},
1314 {0x0000007e, 0x044408a8},
1315 {0x0000007f, 0x00000003},
1316 {0x00000080, 0x00000000},
1317 {0x00000081, 0x01000000},
1318 {0x00000082, 0x02000000},
1319 {0x00000083, 0x00000000},
1320 {0x00000084, 0xe3f3e4f4},
1321 {0x00000085, 0x00052024},
1322 {0x00000087, 0x00000000},
1323 {0x00000088, 0x66036603},
1324 {0x00000089, 0x01000000},
1325 {0x0000008b, 0x1c0a0000},
1326 {0x0000008c, 0xff010000},
1327 {0x0000008e, 0xffffefff},
1328 {0x0000008f, 0xfff3efff},
1329 {0x00000090, 0xfff3efbf},
1330 {0x00000094, 0x00101101},
1331 {0x00000095, 0x00000fff},
1332 {0x00000096, 0x00116fff},
1333 {0x00000097, 0x60010000},
1334 {0x00000098, 0x10010000},
1335 {0x00000099, 0x00006000},
1336 {0x0000009a, 0x00001000},
1337 {0x0000009f, 0x00a47400}
1338};
1339
1340static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1341 {0x0000006f, 0x03044000},
1342 {0x00000070, 0x0480c018},
1343 {0x00000071, 0x00000040},
1344 {0x00000072, 0x01000000},
1345 {0x00000074, 0x000000ff},
1346 {0x00000075, 0x00143400},
1347 {0x00000076, 0x08ec0800},
1348 {0x00000077, 0x040000cc},
1349 {0x00000079, 0x00000000},
1350 {0x0000007a, 0x21000409},
1351 {0x0000007c, 0x00000000},
1352 {0x0000007d, 0xe8000000},
1353 {0x0000007e, 0x044408a8},
1354 {0x0000007f, 0x00000003},
1355 {0x00000080, 0x00000000},
1356 {0x00000081, 0x01000000},
1357 {0x00000082, 0x02000000},
1358 {0x00000083, 0x00000000},
1359 {0x00000084, 0xe3f3e4f4},
1360 {0x00000085, 0x00052024},
1361 {0x00000087, 0x00000000},
1362 {0x00000088, 0x66036603},
1363 {0x00000089, 0x01000000},
1364 {0x0000008b, 0x1c0a0000},
1365 {0x0000008c, 0xff010000},
1366 {0x0000008e, 0xffffefff},
1367 {0x0000008f, 0xfff3efff},
1368 {0x00000090, 0xfff3efbf},
1369 {0x00000094, 0x00101101},
1370 {0x00000095, 0x00000fff},
1371 {0x00000096, 0x00116fff},
1372 {0x00000097, 0x60010000},
1373 {0x00000098, 0x10010000},
1374 {0x00000099, 0x00006000},
1375 {0x0000009a, 0x00001000},
1376 {0x0000009f, 0x00a37400}
1377};
1378
bcc7f5d2
AD		 1379static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1380 {0x0000006f, 0x03044000},
1381 {0x00000070, 0x0480c018},
1382 {0x00000071, 0x00000040},
1383 {0x00000072, 0x01000000},
1384 {0x00000074, 0x000000ff},
1385 {0x00000075, 0x00143400},
1386 {0x00000076, 0x08ec0800},
1387 {0x00000077, 0x040000cc},
1388 {0x00000079, 0x00000000},
1389 {0x0000007a, 0x21000409},
1390 {0x0000007c, 0x00000000},
1391 {0x0000007d, 0xe8000000},
1392 {0x0000007e, 0x044408a8},
1393 {0x0000007f, 0x00000003},
1394 {0x00000080, 0x00000000},
1395 {0x00000081, 0x01000000},
1396 {0x00000082, 0x02000000},
1397 {0x00000083, 0x00000000},
1398 {0x00000084, 0xe3f3e4f4},
1399 {0x00000085, 0x00052024},
1400 {0x00000087, 0x00000000},
1401 {0x00000088, 0x66036603},
1402 {0x00000089, 0x01000000},
1403 {0x0000008b, 0x1c0a0000},
1404 {0x0000008c, 0xff010000},
1405 {0x0000008e, 0xffffefff},
1406 {0x0000008f, 0xfff3efff},
1407 {0x00000090, 0xfff3efbf},
1408 {0x00000094, 0x00101101},
1409 {0x00000095, 0x00000fff},
1410 {0x00000096, 0x00116fff},
1411 {0x00000097, 0x60010000},
1412 {0x00000098, 0x10010000},
1413 {0x00000099, 0x00006000},
1414 {0x0000009a, 0x00001000},
1415 {0x0000009f, 0x00a17730}
1416};
1417
c04c00b4
AD		 1418static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1419 {0x0000006f, 0x03044000},
1420 {0x00000070, 0x0480c018},
1421 {0x00000071, 0x00000040},
1422 {0x00000072, 0x01000000},
1423 {0x00000074, 0x000000ff},
1424 {0x00000075, 0x00143400},
1425 {0x00000076, 0x08ec0800},
1426 {0x00000077, 0x040000cc},
1427 {0x00000079, 0x00000000},
1428 {0x0000007a, 0x21000409},
1429 {0x0000007c, 0x00000000},
1430 {0x0000007d, 0xe8000000},
1431 {0x0000007e, 0x044408a8},
1432 {0x0000007f, 0x00000003},
1433 {0x00000080, 0x00000000},
1434 {0x00000081, 0x01000000},
1435 {0x00000082, 0x02000000},
1436 {0x00000083, 0x00000000},
1437 {0x00000084, 0xe3f3e4f4},
1438 {0x00000085, 0x00052024},
1439 {0x00000087, 0x00000000},
1440 {0x00000088, 0x66036603},
1441 {0x00000089, 0x01000000},
1442 {0x0000008b, 0x1c0a0000},
1443 {0x0000008c, 0xff010000},
1444 {0x0000008e, 0xffffefff},
1445 {0x0000008f, 0xfff3efff},
1446 {0x00000090, 0xfff3efbf},
1447 {0x00000094, 0x00101101},
1448 {0x00000095, 0x00000fff},
1449 {0x00000096, 0x00116fff},
1450 {0x00000097, 0x60010000},
1451 {0x00000098, 0x10010000},
1452 {0x00000099, 0x00006000},
1453 {0x0000009a, 0x00001000},
1454 {0x0000009f, 0x00a07730}
1455};
1456
8b074dd6
AD		 1457/* ucode loading */
1458static int si_mc_load_microcode(struct radeon_device *rdev)
1459{
1460 const __be32 *fw_data;
1461 u32 running, blackout = 0;
1462 u32 *io_mc_regs;
1463 int i, ucode_size, regs_size;
1464
1465 if (!rdev->mc_fw)
1466 return -EINVAL;
1467
1468 switch (rdev->family) {
1469 case CHIP_TAHITI:
1470 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1471 ucode_size = SI_MC_UCODE_SIZE;
1472 regs_size = TAHITI_IO_MC_REGS_SIZE;
1473 break;
1474 case CHIP_PITCAIRN:
1475 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1476 ucode_size = SI_MC_UCODE_SIZE;
1477 regs_size = TAHITI_IO_MC_REGS_SIZE;
1478 break;
1479 case CHIP_VERDE:
1480 default:
1481 io_mc_regs = (u32 *)&verde_io_mc_regs;
1482 ucode_size = SI_MC_UCODE_SIZE;
1483 regs_size = TAHITI_IO_MC_REGS_SIZE;
1484 break;
bcc7f5d2
AD		 1485 case CHIP_OLAND:
1486 io_mc_regs = (u32 *)&oland_io_mc_regs;
1487 ucode_size = OLAND_MC_UCODE_SIZE;
1488 regs_size = TAHITI_IO_MC_REGS_SIZE;
1489 break;
c04c00b4
AD		 1490 case CHIP_HAINAN:
1491 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1492 ucode_size = OLAND_MC_UCODE_SIZE;
1493 regs_size = TAHITI_IO_MC_REGS_SIZE;
1494 break;
8b074dd6
AD		 1495 }
1496
1497 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1498
1499 if (running == 0) {
1500 if (running) {
1501 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1502 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1503 }
1504
1505 /* reset the engine and set to writable */
1506 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1507 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1508
1509 /* load mc io regs */
1510 for (i = 0; i < regs_size; i++) {
1511 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1512 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1513 }
1514 /* load the MC ucode */
1515 fw_data = (const __be32 *)rdev->mc_fw->data;
1516 for (i = 0; i < ucode_size; i++)
1517 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1518
1519 /* put the engine back into the active state */
1520 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1521 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1522 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1523
1524 /* wait for training to complete */
1525 for (i = 0; i < rdev->usec_timeout; i++) {
1526 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1527 break;
1528 udelay(1);
1529 }
1530 for (i = 0; i < rdev->usec_timeout; i++) {
1531 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1532 break;
1533 udelay(1);
1534 }
1535
1536 if (running)
1537 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1538 }
1539
1540 return 0;
1541}
1542
0f0de06c
AD		 1543static int si_init_microcode(struct radeon_device *rdev)
1544{
0f0de06c
AD		 1545 const char *chip_name;
1546 const char *rlc_chip_name;
1547 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
a9e61410 1548 size_t smc_req_size;
0f0de06c
AD		 1549 char fw_name[30];
1550 int err;
1551
1552 DRM_DEBUG("\n");
1553
0f0de06c
AD		 1554 switch (rdev->family) {
1555 case CHIP_TAHITI:
1556 chip_name = "TAHITI";
1557 rlc_chip_name = "TAHITI";
1558 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1559 me_req_size = SI_PM4_UCODE_SIZE * 4;
1560 ce_req_size = SI_CE_UCODE_SIZE * 4;
1561 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1562 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1563 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1564 break;
1565 case CHIP_PITCAIRN:
1566 chip_name = "PITCAIRN";
1567 rlc_chip_name = "PITCAIRN";
1568 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1569 me_req_size = SI_PM4_UCODE_SIZE * 4;
1570 ce_req_size = SI_CE_UCODE_SIZE * 4;
1571 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1572 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1573 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1574 break;
1575 case CHIP_VERDE:
1576 chip_name = "VERDE";
1577 rlc_chip_name = "VERDE";
1578 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1579 me_req_size = SI_PM4_UCODE_SIZE * 4;
1580 ce_req_size = SI_CE_UCODE_SIZE * 4;
1581 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1582 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1583 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
0f0de06c 1584 break;
bcc7f5d2
AD		 1585 case CHIP_OLAND:
1586 chip_name = "OLAND";
1587 rlc_chip_name = "OLAND";
1588 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1589 me_req_size = SI_PM4_UCODE_SIZE * 4;
1590 ce_req_size = SI_CE_UCODE_SIZE * 4;
1591 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1592 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1593 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
bcc7f5d2 1594 break;
c04c00b4
AD		 1595 case CHIP_HAINAN:
1596 chip_name = "HAINAN";
1597 rlc_chip_name = "HAINAN";
1598 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1599 me_req_size = SI_PM4_UCODE_SIZE * 4;
1600 ce_req_size = SI_CE_UCODE_SIZE * 4;
1601 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1602 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1603 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
c04c00b4 1604 break;
0f0de06c
AD		 1605 default: BUG();
1606 }
1607
1608 DRM_INFO("Loading %s Microcode\n", chip_name);
1609
1610 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1611 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
0f0de06c
AD		 1612 if (err)
1613 goto out;
1614 if (rdev->pfp_fw->size != pfp_req_size) {
1615 printk(KERN_ERR
1616 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1617 rdev->pfp_fw->size, fw_name);
1618 err = -EINVAL;
1619 goto out;
1620 }
1621
1622 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1623 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
0f0de06c
AD		 1624 if (err)
1625 goto out;
1626 if (rdev->me_fw->size != me_req_size) {
1627 printk(KERN_ERR
1628 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1629 rdev->me_fw->size, fw_name);
1630 err = -EINVAL;
1631 }
1632
1633 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1634 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
0f0de06c
AD		 1635 if (err)
1636 goto out;
1637 if (rdev->ce_fw->size != ce_req_size) {
1638 printk(KERN_ERR
1639 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1640 rdev->ce_fw->size, fw_name);
1641 err = -EINVAL;
1642 }
1643
1644 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
0a168933 1645 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1646 if (err)
1647 goto out;
1648 if (rdev->rlc_fw->size != rlc_req_size) {
1649 printk(KERN_ERR
1650 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1651 rdev->rlc_fw->size, fw_name);
1652 err = -EINVAL;
1653 }
1654
1655 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1656 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1657 if (err)
1658 goto out;
1659 if (rdev->mc_fw->size != mc_req_size) {
1660 printk(KERN_ERR
1661 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1662 rdev->mc_fw->size, fw_name);
1663 err = -EINVAL;
1664 }
1665
a9e61410 1666 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
0a168933 1667 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
8a53fa23
AD		 1668 if (err) {
1669 printk(KERN_ERR
1670 "smc: error loading firmware \"%s\"\n",
1671 fw_name);
1672 release_firmware(rdev->smc_fw);
1673 rdev->smc_fw = NULL;
1674 } else if (rdev->smc_fw->size != smc_req_size) {
a9e61410
AD		 1675 printk(KERN_ERR
1676 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1677 rdev->smc_fw->size, fw_name);
1678 err = -EINVAL;
1679 }
1680
0f0de06c 1681out:
0f0de06c
AD		 1682 if (err) {
1683 if (err != -EINVAL)
1684 printk(KERN_ERR
1685 "si_cp: Failed to load firmware \"%s\"\n",
1686 fw_name);
1687 release_firmware(rdev->pfp_fw);
1688 rdev->pfp_fw = NULL;
1689 release_firmware(rdev->me_fw);
1690 rdev->me_fw = NULL;
1691 release_firmware(rdev->ce_fw);
1692 rdev->ce_fw = NULL;
1693 release_firmware(rdev->rlc_fw);
1694 rdev->rlc_fw = NULL;
1695 release_firmware(rdev->mc_fw);
1696 rdev->mc_fw = NULL;
a9e61410
AD		 1697 release_firmware(rdev->smc_fw);
1698 rdev->smc_fw = NULL;
0f0de06c
AD		 1699 }
1700 return err;
1701}
1702
43b3cd99
AD		 1703/* watermark setup */
1704static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1705 struct radeon_crtc *radeon_crtc,
1706 struct drm_display_mode *mode,
1707 struct drm_display_mode *other_mode)
1708{
1709 u32 tmp;
1710 /*
1711 * Line Buffer Setup
1712 * There are 3 line buffers, each one shared by 2 display controllers.
1713 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1714 * the display controllers. The paritioning is done via one of four
1715 * preset allocations specified in bits 21:20:
1716 * 0 - half lb
1717 * 2 - whole lb, other crtc must be disabled
1718 */
1719 /* this can get tricky if we have two large displays on a paired group
1720 * of crtcs. Ideally for multiple large displays we'd assign them to
1721 * non-linked crtcs for maximum line buffer allocation.
1722 */
1723 if (radeon_crtc->base.enabled && mode) {
1724 if (other_mode)
1725 tmp = 0; /* 1/2 */
1726 else
1727 tmp = 2; /* whole */
1728 } else
1729 tmp = 0;
1730
1731 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1732 DC_LB_MEMORY_CONFIG(tmp));
1733
1734 if (radeon_crtc->base.enabled && mode) {
1735 switch (tmp) {
1736 case 0:
1737 default:
1738 return 4096 * 2;
1739 case 2:
1740 return 8192 * 2;
1741 }
1742 }
1743
1744 /* controller not enabled, so no lb used */
1745 return 0;
1746}
1747
ca7db22b 1748static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1749{
1750 u32 tmp = RREG32(MC_SHARED_CHMAP);
1751
1752 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1753 case 0:
1754 default:
1755 return 1;
1756 case 1:
1757 return 2;
1758 case 2:
1759 return 4;
1760 case 3:
1761 return 8;
1762 case 4:
1763 return 3;
1764 case 5:
1765 return 6;
1766 case 6:
1767 return 10;
1768 case 7:
1769 return 12;
1770 case 8:
1771 return 16;
1772 }
1773}
1774
1775struct dce6_wm_params {
1776 u32 dram_channels; /* number of dram channels */
1777 u32 yclk; /* bandwidth per dram data pin in kHz */
1778 u32 sclk; /* engine clock in kHz */
1779 u32 disp_clk; /* display clock in kHz */
1780 u32 src_width; /* viewport width */
1781 u32 active_time; /* active display time in ns */
1782 u32 blank_time; /* blank time in ns */
1783 bool interlaced; /* mode is interlaced */
1784 fixed20_12 vsc; /* vertical scale ratio */
1785 u32 num_heads; /* number of active crtcs */
1786 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1787 u32 lb_size; /* line buffer allocated to pipe */
1788 u32 vtaps; /* vertical scaler taps */
1789};
1790
1791static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1792{
1793 /* Calculate raw DRAM Bandwidth */
1794 fixed20_12 dram_efficiency; /* 0.7 */
1795 fixed20_12 yclk, dram_channels, bandwidth;
1796 fixed20_12 a;
1797
1798 a.full = dfixed_const(1000);
1799 yclk.full = dfixed_const(wm->yclk);
1800 yclk.full = dfixed_div(yclk, a);
1801 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1802 a.full = dfixed_const(10);
1803 dram_efficiency.full = dfixed_const(7);
1804 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1805 bandwidth.full = dfixed_mul(dram_channels, yclk);
1806 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1807
1808 return dfixed_trunc(bandwidth);
1809}
1810
1811static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1812{
1813 /* Calculate DRAM Bandwidth and the part allocated to display. */
1814 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1815 fixed20_12 yclk, dram_channels, bandwidth;
1816 fixed20_12 a;
1817
1818 a.full = dfixed_const(1000);
1819 yclk.full = dfixed_const(wm->yclk);
1820 yclk.full = dfixed_div(yclk, a);
1821 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1822 a.full = dfixed_const(10);
1823 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1824 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1825 bandwidth.full = dfixed_mul(dram_channels, yclk);
1826 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1827
1828 return dfixed_trunc(bandwidth);
1829}
1830
1831static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1832{
1833 /* Calculate the display Data return Bandwidth */
1834 fixed20_12 return_efficiency; /* 0.8 */
1835 fixed20_12 sclk, bandwidth;
1836 fixed20_12 a;
1837
1838 a.full = dfixed_const(1000);
1839 sclk.full = dfixed_const(wm->sclk);
1840 sclk.full = dfixed_div(sclk, a);
1841 a.full = dfixed_const(10);
1842 return_efficiency.full = dfixed_const(8);
1843 return_efficiency.full = dfixed_div(return_efficiency, a);
1844 a.full = dfixed_const(32);
1845 bandwidth.full = dfixed_mul(a, sclk);
1846 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1847
1848 return dfixed_trunc(bandwidth);
1849}
1850
1851static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1852{
1853 return 32;
1854}
1855
1856static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1857{
1858 /* Calculate the DMIF Request Bandwidth */
1859 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1860 fixed20_12 disp_clk, sclk, bandwidth;
1861 fixed20_12 a, b1, b2;
1862 u32 min_bandwidth;
1863
1864 a.full = dfixed_const(1000);
1865 disp_clk.full = dfixed_const(wm->disp_clk);
1866 disp_clk.full = dfixed_div(disp_clk, a);
1867 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1868 b1.full = dfixed_mul(a, disp_clk);
1869
1870 a.full = dfixed_const(1000);
1871 sclk.full = dfixed_const(wm->sclk);
1872 sclk.full = dfixed_div(sclk, a);
1873 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1874 b2.full = dfixed_mul(a, sclk);
1875
1876 a.full = dfixed_const(10);
1877 disp_clk_request_efficiency.full = dfixed_const(8);
1878 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1879
1880 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1881
1882 a.full = dfixed_const(min_bandwidth);
1883 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1884
1885 return dfixed_trunc(bandwidth);
1886}
1887
1888static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1889{
1890 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1891 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1892 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1893 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1894
1895 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1896}
1897
1898static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1899{
1900 /* Calculate the display mode Average Bandwidth
1901 * DisplayMode should contain the source and destination dimensions,
1902 * timing, etc.
1903 */
1904 fixed20_12 bpp;
1905 fixed20_12 line_time;
1906 fixed20_12 src_width;
1907 fixed20_12 bandwidth;
1908 fixed20_12 a;
1909
1910 a.full = dfixed_const(1000);
1911 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1912 line_time.full = dfixed_div(line_time, a);
1913 bpp.full = dfixed_const(wm->bytes_per_pixel);
1914 src_width.full = dfixed_const(wm->src_width);
1915 bandwidth.full = dfixed_mul(src_width, bpp);
1916 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1917 bandwidth.full = dfixed_div(bandwidth, line_time);
1918
1919 return dfixed_trunc(bandwidth);
1920}
1921
1922static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1923{
1924 /* First calcualte the latency in ns */
1925 u32 mc_latency = 2000; /* 2000 ns. */
1926 u32 available_bandwidth = dce6_available_bandwidth(wm);
1927 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1928 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1929 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1930 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1931 (wm->num_heads * cursor_line_pair_return_time);
1932 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1933 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1934 u32 tmp, dmif_size = 12288;
1935 fixed20_12 a, b, c;
1936
1937 if (wm->num_heads == 0)
1938 return 0;
1939
1940 a.full = dfixed_const(2);
1941 b.full = dfixed_const(1);
1942 if ((wm->vsc.full > a.full) ||
1943 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1944 (wm->vtaps >= 5) ||
1945 ((wm->vsc.full >= a.full) && wm->interlaced))
1946 max_src_lines_per_dst_line = 4;
1947 else
1948 max_src_lines_per_dst_line = 2;
1949
1950 a.full = dfixed_const(available_bandwidth);
1951 b.full = dfixed_const(wm->num_heads);
1952 a.full = dfixed_div(a, b);
1953
1954 b.full = dfixed_const(mc_latency + 512);
1955 c.full = dfixed_const(wm->disp_clk);
1956 b.full = dfixed_div(b, c);
1957
1958 c.full = dfixed_const(dmif_size);
1959 b.full = dfixed_div(c, b);
1960
1961 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1962
1963 b.full = dfixed_const(1000);
1964 c.full = dfixed_const(wm->disp_clk);
1965 b.full = dfixed_div(c, b);
1966 c.full = dfixed_const(wm->bytes_per_pixel);
1967 b.full = dfixed_mul(b, c);
1968
1969 lb_fill_bw = min(tmp, dfixed_trunc(b));
1970
1971 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1972 b.full = dfixed_const(1000);
1973 c.full = dfixed_const(lb_fill_bw);
1974 b.full = dfixed_div(c, b);
1975 a.full = dfixed_div(a, b);
1976 line_fill_time = dfixed_trunc(a);
1977
1978 if (line_fill_time < wm->active_time)
1979 return latency;
1980 else
1981 return latency + (line_fill_time - wm->active_time);
1982
1983}
1984
1985static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1986{
1987 if (dce6_average_bandwidth(wm) <=
1988 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1989 return true;
1990 else
1991 return false;
1992};
1993
1994static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
1995{
1996 if (dce6_average_bandwidth(wm) <=
1997 (dce6_available_bandwidth(wm) / wm->num_heads))
1998 return true;
1999 else
2000 return false;
2001};
2002
2003static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2004{
2005 u32 lb_partitions = wm->lb_size / wm->src_width;
2006 u32 line_time = wm->active_time + wm->blank_time;
2007 u32 latency_tolerant_lines;
2008 u32 latency_hiding;
2009 fixed20_12 a;
2010
2011 a.full = dfixed_const(1);
2012 if (wm->vsc.full > a.full)
2013 latency_tolerant_lines = 1;
2014 else {
2015 if (lb_partitions <= (wm->vtaps + 1))
2016 latency_tolerant_lines = 1;
2017 else
2018 latency_tolerant_lines = 2;
2019 }
2020
2021 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2022
2023 if (dce6_latency_watermark(wm) <= latency_hiding)
2024 return true;
2025 else
2026 return false;
2027}
2028
2029static void dce6_program_watermarks(struct radeon_device *rdev,
2030 struct radeon_crtc *radeon_crtc,
2031 u32 lb_size, u32 num_heads)
2032{
2033 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2034 struct dce6_wm_params wm_low, wm_high;
2035 u32 dram_channels;
43b3cd99
AD		 2036 u32 pixel_period;
2037 u32 line_time = 0;
2038 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2039 u32 priority_a_mark = 0, priority_b_mark = 0;
2040 u32 priority_a_cnt = PRIORITY_OFF;
2041 u32 priority_b_cnt = PRIORITY_OFF;
2042 u32 tmp, arb_control3;
2043 fixed20_12 a, b, c;
2044
2045 if (radeon_crtc->base.enabled && num_heads && mode) {
2046 pixel_period = 1000000 / (u32)mode->clock;
2047 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2048 priority_a_cnt = 0;
2049 priority_b_cnt = 0;
2050
ca7db22b 2051 if (rdev->family == CHIP_ARUBA)
c696e53f 2052 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2053 else
c696e53f
AD		 2054 dram_channels = si_get_number_of_dram_channels(rdev);
2055
2056 /* watermark for high clocks */
2057 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2058 wm_high.yclk =
2059 radeon_dpm_get_mclk(rdev, false) * 10;
2060 wm_high.sclk =
2061 radeon_dpm_get_sclk(rdev, false) * 10;
2062 } else {
2063 wm_high.yclk = rdev->pm.current_mclk * 10;
2064 wm_high.sclk = rdev->pm.current_sclk * 10;
2065 }
2066
2067 wm_high.disp_clk = mode->clock;
2068 wm_high.src_width = mode->crtc_hdisplay;
2069 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2070 wm_high.blank_time = line_time - wm_high.active_time;
2071 wm_high.interlaced = false;
2072 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2073 wm_high.interlaced = true;
2074 wm_high.vsc = radeon_crtc->vsc;
2075 wm_high.vtaps = 1;
2076 if (radeon_crtc->rmx_type != RMX_OFF)
2077 wm_high.vtaps = 2;
2078 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2079 wm_high.lb_size = lb_size;
2080 wm_high.dram_channels = dram_channels;
2081 wm_high.num_heads = num_heads;
2082
2083 /* watermark for low clocks */
2084 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2085 wm_low.yclk =
2086 radeon_dpm_get_mclk(rdev, true) * 10;
2087 wm_low.sclk =
2088 radeon_dpm_get_sclk(rdev, true) * 10;
2089 } else {
2090 wm_low.yclk = rdev->pm.current_mclk * 10;
2091 wm_low.sclk = rdev->pm.current_sclk * 10;
2092 }
2093
2094 wm_low.disp_clk = mode->clock;
2095 wm_low.src_width = mode->crtc_hdisplay;
2096 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2097 wm_low.blank_time = line_time - wm_low.active_time;
2098 wm_low.interlaced = false;
2099 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2100 wm_low.interlaced = true;
2101 wm_low.vsc = radeon_crtc->vsc;
2102 wm_low.vtaps = 1;
2103 if (radeon_crtc->rmx_type != RMX_OFF)
2104 wm_low.vtaps = 2;
2105 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2106 wm_low.lb_size = lb_size;
2107 wm_low.dram_channels = dram_channels;
2108 wm_low.num_heads = num_heads;
43b3cd99
AD		 2109
2110 /* set for high clocks */
c696e53f 2111 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2112 /* set for low clocks */
c696e53f 2113 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2114
2115 /* possibly force display priority to high */
2116 /* should really do this at mode validation time... */
c696e53f
AD		 2117 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2118 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2119 !dce6_check_latency_hiding(&wm_high) ||
2120 (rdev->disp_priority == 2)) {
2121 DRM_DEBUG_KMS("force priority to high\n");
2122 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2123 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2124 }
2125 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2126 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2127 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2128 (rdev->disp_priority == 2)) {
2129 DRM_DEBUG_KMS("force priority to high\n");
2130 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2131 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2132 }
2133
2134 a.full = dfixed_const(1000);
2135 b.full = dfixed_const(mode->clock);
2136 b.full = dfixed_div(b, a);
2137 c.full = dfixed_const(latency_watermark_a);
2138 c.full = dfixed_mul(c, b);
2139 c.full = dfixed_mul(c, radeon_crtc->hsc);
2140 c.full = dfixed_div(c, a);
2141 a.full = dfixed_const(16);
2142 c.full = dfixed_div(c, a);
2143 priority_a_mark = dfixed_trunc(c);
2144 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2145
2146 a.full = dfixed_const(1000);
2147 b.full = dfixed_const(mode->clock);
2148 b.full = dfixed_div(b, a);
2149 c.full = dfixed_const(latency_watermark_b);
2150 c.full = dfixed_mul(c, b);
2151 c.full = dfixed_mul(c, radeon_crtc->hsc);
2152 c.full = dfixed_div(c, a);
2153 a.full = dfixed_const(16);
2154 c.full = dfixed_div(c, a);
2155 priority_b_mark = dfixed_trunc(c);
2156 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2157 }
2158
2159 /* select wm A */
2160 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2161 tmp = arb_control3;
2162 tmp &= ~LATENCY_WATERMARK_MASK(3);
2163 tmp |= LATENCY_WATERMARK_MASK(1);
2164 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2165 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2166 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2167 LATENCY_HIGH_WATERMARK(line_time)));
2168 /* select wm B */
2169 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2170 tmp &= ~LATENCY_WATERMARK_MASK(3);
2171 tmp |= LATENCY_WATERMARK_MASK(2);
2172 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2173 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2174 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2175 LATENCY_HIGH_WATERMARK(line_time)));
2176 /* restore original selection */
2177 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2178
2179 /* write the priority marks */
2180 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2181 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2182
7178d2a6
AD		 2183 /* save values for DPM */
2184 radeon_crtc->line_time = line_time;
2185 radeon_crtc->wm_high = latency_watermark_a;
2186 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2187}
2188
2189void dce6_bandwidth_update(struct radeon_device *rdev)
2190{
2191 struct drm_display_mode *mode0 = NULL;
2192 struct drm_display_mode *mode1 = NULL;
2193 u32 num_heads = 0, lb_size;
2194 int i;
2195
2196 radeon_update_display_priority(rdev);
2197
2198 for (i = 0; i < rdev->num_crtc; i++) {
2199 if (rdev->mode_info.crtcs[i]->base.enabled)
2200 num_heads++;
2201 }
2202 for (i = 0; i < rdev->num_crtc; i += 2) {
2203 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2204 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2205 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2206 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2207 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2208 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2209 }
2210}
2211
0a96d72b
AD		 2212/*
2213 * Core functions
2214 */
0a96d72b
AD		 2215static void si_tiling_mode_table_init(struct radeon_device *rdev)
2216{
2217 const u32 num_tile_mode_states = 32;
2218 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2219
2220 switch (rdev->config.si.mem_row_size_in_kb) {
2221 case 1:
2222 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2223 break;
2224 case 2:
2225 default:
2226 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2227 break;
2228 case 4:
2229 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2230 break;
2231 }
2232
2233 if ((rdev->family == CHIP_TAHITI) ||
2234 (rdev->family == CHIP_PITCAIRN)) {
2235 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2236 switch (reg_offset) {
2237 case 0: /* non-AA compressed depth or any compressed stencil */
2238 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2239 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2240 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2241 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2242 NUM_BANKS(ADDR_SURF_16_BANK) |
2243 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2244 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2245 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2246 break;
2247 case 1: /* 2xAA/4xAA compressed depth only */
2248 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2249 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2250 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2251 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2252 NUM_BANKS(ADDR_SURF_16_BANK) |
2253 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2254 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2255 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2256 break;
2257 case 2: /* 8xAA compressed depth only */
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2261 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2262 NUM_BANKS(ADDR_SURF_16_BANK) |
2263 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2264 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2265 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2266 break;
2267 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2269 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2270 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2271 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2272 NUM_BANKS(ADDR_SURF_16_BANK) |
2273 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2274 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2275 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2276 break;
2277 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2279 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2282 NUM_BANKS(ADDR_SURF_16_BANK) |
2283 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2284 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2285 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2286 break;
2287 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2288 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2289 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2290 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2291 TILE_SPLIT(split_equal_to_row_size) |
2292 NUM_BANKS(ADDR_SURF_16_BANK) |
2293 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2294 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2295 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2296 break;
2297 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2298 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2299 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2300 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2301 TILE_SPLIT(split_equal_to_row_size) |
2302 NUM_BANKS(ADDR_SURF_16_BANK) |
2303 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2304 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2305 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2306 break;
2307 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2308 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2310 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2311 TILE_SPLIT(split_equal_to_row_size) |
2312 NUM_BANKS(ADDR_SURF_16_BANK) |
2313 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2314 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2315 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2316 break;
2317 case 8: /* 1D and 1D Array Surfaces */
2318 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2319 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2320 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2321 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2322 NUM_BANKS(ADDR_SURF_16_BANK) |
2323 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2324 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2325 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2326 break;
2327 case 9: /* Displayable maps. */
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2329 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2331 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2332 NUM_BANKS(ADDR_SURF_16_BANK) |
2333 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2334 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2335 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2336 break;
2337 case 10: /* Display 8bpp. */
2338 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2339 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2340 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2341 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2342 NUM_BANKS(ADDR_SURF_16_BANK) |
2343 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2346 break;
2347 case 11: /* Display 16bpp. */
2348 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2349 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2350 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2351 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2352 NUM_BANKS(ADDR_SURF_16_BANK) |
2353 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2354 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2355 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2356 break;
2357 case 12: /* Display 32bpp. */
2358 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2359 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2360 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2361 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2362 NUM_BANKS(ADDR_SURF_16_BANK) |
2363 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2364 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2365 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2366 break;
2367 case 13: /* Thin. */
2368 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2369 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2370 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2371 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2372 NUM_BANKS(ADDR_SURF_16_BANK) |
2373 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2376 break;
2377 case 14: /* Thin 8 bpp. */
2378 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2379 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2380 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2381 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2382 NUM_BANKS(ADDR_SURF_16_BANK) |
2383 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2384 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2385 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2386 break;
2387 case 15: /* Thin 16 bpp. */
2388 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2389 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2390 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2391 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2392 NUM_BANKS(ADDR_SURF_16_BANK) |
2393 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2394 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2395 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2396 break;
2397 case 16: /* Thin 32 bpp. */
2398 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2399 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2400 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2401 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2402 NUM_BANKS(ADDR_SURF_16_BANK) |
2403 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2406 break;
2407 case 17: /* Thin 64 bpp. */
2408 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2409 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2410 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2411 TILE_SPLIT(split_equal_to_row_size) |
2412 NUM_BANKS(ADDR_SURF_16_BANK) |
2413 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2414 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2415 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2416 break;
2417 case 21: /* 8 bpp PRT. */
2418 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2419 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2420 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2421 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2422 NUM_BANKS(ADDR_SURF_16_BANK) |
2423 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2424 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2425 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2426 break;
2427 case 22: /* 16 bpp PRT */
2428 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2429 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2430 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2431 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2432 NUM_BANKS(ADDR_SURF_16_BANK) |
2433 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2434 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2435 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2436 break;
2437 case 23: /* 32 bpp PRT */
2438 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2439 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2440 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2441 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2442 NUM_BANKS(ADDR_SURF_16_BANK) |
2443 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2446 break;
2447 case 24: /* 64 bpp PRT */
2448 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2449 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2450 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2451 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2452 NUM_BANKS(ADDR_SURF_16_BANK) |
2453 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2454 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2455 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2456 break;
2457 case 25: /* 128 bpp PRT */
2458 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2459 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2460 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2461 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2462 NUM_BANKS(ADDR_SURF_8_BANK) |
2463 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2466 break;
2467 default:
2468 gb_tile_moden = 0;
2469 break;
2470 }
64d7b8be 2471 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2472 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2473 }
d0ae7fcc 2474 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2475 (rdev->family == CHIP_OLAND) ||
2476 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2477 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2478 switch (reg_offset) {
2479 case 0: /* non-AA compressed depth or any compressed stencil */
2480 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2481 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2482 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2483 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2484 NUM_BANKS(ADDR_SURF_16_BANK) |
2485 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2488 break;
2489 case 1: /* 2xAA/4xAA compressed depth only */
2490 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2491 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2492 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2493 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2494 NUM_BANKS(ADDR_SURF_16_BANK) |
2495 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2496 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2497 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2498 break;
2499 case 2: /* 8xAA compressed depth only */
2500 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2501 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2502 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2503 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2504 NUM_BANKS(ADDR_SURF_16_BANK) |
2505 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2508 break;
2509 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2510 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2511 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2512 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2513 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2514 NUM_BANKS(ADDR_SURF_16_BANK) |
2515 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2516 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2517 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2518 break;
2519 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2520 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2521 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2522 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2523 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2524 NUM_BANKS(ADDR_SURF_16_BANK) |
2525 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2526 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2527 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2528 break;
2529 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2530 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2531 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2532 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2533 TILE_SPLIT(split_equal_to_row_size) |
2534 NUM_BANKS(ADDR_SURF_16_BANK) |
2535 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2536 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2537 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2538 break;
2539 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2540 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2541 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2542 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2543 TILE_SPLIT(split_equal_to_row_size) |
2544 NUM_BANKS(ADDR_SURF_16_BANK) |
2545 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2546 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2547 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2548 break;
2549 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2550 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2551 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2552 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2553 TILE_SPLIT(split_equal_to_row_size) |
2554 NUM_BANKS(ADDR_SURF_16_BANK) |
2555 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2556 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2557 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2558 break;
2559 case 8: /* 1D and 1D Array Surfaces */
2560 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2561 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2562 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2563 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2564 NUM_BANKS(ADDR_SURF_16_BANK) |
2565 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2566 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2567 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2568 break;
2569 case 9: /* Displayable maps. */
2570 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2571 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2572 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2573 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2574 NUM_BANKS(ADDR_SURF_16_BANK) |
2575 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2576 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2577 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2578 break;
2579 case 10: /* Display 8bpp. */
2580 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2581 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2582 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2583 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2584 NUM_BANKS(ADDR_SURF_16_BANK) |
2585 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2586 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2587 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2588 break;
2589 case 11: /* Display 16bpp. */
2590 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2591 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2592 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2593 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2594 NUM_BANKS(ADDR_SURF_16_BANK) |
2595 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2596 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2597 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2598 break;
2599 case 12: /* Display 32bpp. */
2600 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2601 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2602 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2603 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2604 NUM_BANKS(ADDR_SURF_16_BANK) |
2605 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2606 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2607 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2608 break;
2609 case 13: /* Thin. */
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2611 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2612 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2613 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2614 NUM_BANKS(ADDR_SURF_16_BANK) |
2615 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2616 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2617 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2618 break;
2619 case 14: /* Thin 8 bpp. */
2620 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2621 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2622 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2623 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2624 NUM_BANKS(ADDR_SURF_16_BANK) |
2625 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2626 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2627 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2628 break;
2629 case 15: /* Thin 16 bpp. */
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2631 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2633 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2634 NUM_BANKS(ADDR_SURF_16_BANK) |
2635 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2636 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2637 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2638 break;
2639 case 16: /* Thin 32 bpp. */
2640 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2643 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2644 NUM_BANKS(ADDR_SURF_16_BANK) |
2645 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2646 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2647 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2648 break;
2649 case 17: /* Thin 64 bpp. */
2650 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2651 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2652 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2653 TILE_SPLIT(split_equal_to_row_size) |
2654 NUM_BANKS(ADDR_SURF_16_BANK) |
2655 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2656 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2657 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2658 break;
2659 case 21: /* 8 bpp PRT. */
2660 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2661 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2662 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2663 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2664 NUM_BANKS(ADDR_SURF_16_BANK) |
2665 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2666 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2667 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2668 break;
2669 case 22: /* 16 bpp PRT */
2670 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2671 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2672 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2673 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2674 NUM_BANKS(ADDR_SURF_16_BANK) |
2675 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2676 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2677 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2678 break;
2679 case 23: /* 32 bpp PRT */
2680 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2681 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2682 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2683 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2684 NUM_BANKS(ADDR_SURF_16_BANK) |
2685 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2686 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2687 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2688 break;
2689 case 24: /* 64 bpp PRT */
2690 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2691 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2692 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2693 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2694 NUM_BANKS(ADDR_SURF_16_BANK) |
2695 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2696 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2697 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2698 break;
2699 case 25: /* 128 bpp PRT */
2700 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2701 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2702 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2703 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2704 NUM_BANKS(ADDR_SURF_8_BANK) |
2705 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2706 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2707 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2708 break;
2709 default:
2710 gb_tile_moden = 0;
2711 break;
2712 }
64d7b8be 2713 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2714 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2715 }
2716 } else
2717 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2718}
2719
1a8ca750
AD		 2720static void si_select_se_sh(struct radeon_device *rdev,
2721 u32 se_num, u32 sh_num)
2722{
2723 u32 data = INSTANCE_BROADCAST_WRITES;
2724
2725 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2726 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2727 else if (se_num == 0xffffffff)
2728 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2729 else if (sh_num == 0xffffffff)
2730 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2731 else
2732 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2733 WREG32(GRBM_GFX_INDEX, data);
2734}
2735
2736static u32 si_create_bitmask(u32 bit_width)
2737{
2738 u32 i, mask = 0;
2739
2740 for (i = 0; i < bit_width; i++) {
2741 mask <<= 1;
2742 mask |= 1;
2743 }
2744 return mask;
2745}
2746
2747static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2748{
2749 u32 data, mask;
2750
2751 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2752 if (data & 1)
2753 data &= INACTIVE_CUS_MASK;
2754 else
2755 data = 0;
2756 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2757
2758 data >>= INACTIVE_CUS_SHIFT;
2759
2760 mask = si_create_bitmask(cu_per_sh);
2761
2762 return ~data & mask;
2763}
2764
2765static void si_setup_spi(struct radeon_device *rdev,
2766 u32 se_num, u32 sh_per_se,
2767 u32 cu_per_sh)
2768{
2769 int i, j, k;
2770 u32 data, mask, active_cu;
2771
2772 for (i = 0; i < se_num; i++) {
2773 for (j = 0; j < sh_per_se; j++) {
2774 si_select_se_sh(rdev, i, j);
2775 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2776 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2777
2778 mask = 1;
2779 for (k = 0; k < 16; k++) {
2780 mask <<= k;
2781 if (active_cu & mask) {
2782 data &= ~mask;
2783 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2784 break;
2785 }
2786 }
2787 }
2788 }
2789 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2790}
2791
2792static u32 si_get_rb_disabled(struct radeon_device *rdev,
2793 u32 max_rb_num, u32 se_num,
2794 u32 sh_per_se)
2795{
2796 u32 data, mask;
2797
2798 data = RREG32(CC_RB_BACKEND_DISABLE);
2799 if (data & 1)
2800 data &= BACKEND_DISABLE_MASK;
2801 else
2802 data = 0;
2803 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2804
2805 data >>= BACKEND_DISABLE_SHIFT;
2806
2807 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2808
2809 return data & mask;
2810}
2811
2812static void si_setup_rb(struct radeon_device *rdev,
2813 u32 se_num, u32 sh_per_se,
2814 u32 max_rb_num)
2815{
2816 int i, j;
2817 u32 data, mask;
2818 u32 disabled_rbs = 0;
2819 u32 enabled_rbs = 0;
2820
2821 for (i = 0; i < se_num; i++) {
2822 for (j = 0; j < sh_per_se; j++) {
2823 si_select_se_sh(rdev, i, j);
2824 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2825 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2826 }
2827 }
2828 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2829
2830 mask = 1;
2831 for (i = 0; i < max_rb_num; i++) {
2832 if (!(disabled_rbs & mask))
2833 enabled_rbs |= mask;
2834 mask <<= 1;
2835 }
2836
2837 for (i = 0; i < se_num; i++) {
2838 si_select_se_sh(rdev, i, 0xffffffff);
2839 data = 0;
2840 for (j = 0; j < sh_per_se; j++) {
2841 switch (enabled_rbs & 3) {
2842 case 1:
2843 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2844 break;
2845 case 2:
2846 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2847 break;
2848 case 3:
2849 default:
2850 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2851 break;
2852 }
2853 enabled_rbs >>= 2;
2854 }
2855 WREG32(PA_SC_RASTER_CONFIG, data);
2856 }
2857 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2858}
2859
0a96d72b
AD		 2860static void si_gpu_init(struct radeon_device *rdev)
2861{
0a96d72b
AD		 2862 u32 gb_addr_config = 0;
2863 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2864 u32 sx_debug_1;
0a96d72b
AD		 2865 u32 hdp_host_path_cntl;
2866 u32 tmp;
2867 int i, j;
2868
2869 switch (rdev->family) {
2870 case CHIP_TAHITI:
2871 rdev->config.si.max_shader_engines = 2;
0a96d72b 2872 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2873 rdev->config.si.max_cu_per_sh = 8;
2874 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2875 rdev->config.si.max_backends_per_se = 4;
2876 rdev->config.si.max_texture_channel_caches = 12;
2877 rdev->config.si.max_gprs = 256;
2878 rdev->config.si.max_gs_threads = 32;
2879 rdev->config.si.max_hw_contexts = 8;
2880
2881 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2882 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2883 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2884 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2885 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2886 break;
2887 case CHIP_PITCAIRN:
2888 rdev->config.si.max_shader_engines = 2;
0a96d72b 2889 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2890 rdev->config.si.max_cu_per_sh = 5;
2891 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2892 rdev->config.si.max_backends_per_se = 4;
2893 rdev->config.si.max_texture_channel_caches = 8;
2894 rdev->config.si.max_gprs = 256;
2895 rdev->config.si.max_gs_threads = 32;
2896 rdev->config.si.max_hw_contexts = 8;
2897
2898 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2899 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2900 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2901 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2902 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2903 break;
2904 case CHIP_VERDE:
2905 default:
2906 rdev->config.si.max_shader_engines = 1;
0a96d72b 2907 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2908 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2909 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2910 rdev->config.si.max_backends_per_se = 4;
2911 rdev->config.si.max_texture_channel_caches = 4;
2912 rdev->config.si.max_gprs = 256;
2913 rdev->config.si.max_gs_threads = 32;
2914 rdev->config.si.max_hw_contexts = 8;
2915
d0ae7fcc
AD		 2916 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2917 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2918 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2919 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2920 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2921 break;
2922 case CHIP_OLAND:
2923 rdev->config.si.max_shader_engines = 1;
2924 rdev->config.si.max_tile_pipes = 4;
2925 rdev->config.si.max_cu_per_sh = 6;
2926 rdev->config.si.max_sh_per_se = 1;
2927 rdev->config.si.max_backends_per_se = 2;
2928 rdev->config.si.max_texture_channel_caches = 4;
2929 rdev->config.si.max_gprs = 256;
2930 rdev->config.si.max_gs_threads = 16;
2931 rdev->config.si.max_hw_contexts = 8;
2932
0a96d72b
AD		 2933 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2934 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2935 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2936 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2937 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2938 break;
8b02859d
AD		 2939 case CHIP_HAINAN:
2940 rdev->config.si.max_shader_engines = 1;
2941 rdev->config.si.max_tile_pipes = 4;
2942 rdev->config.si.max_cu_per_sh = 5;
2943 rdev->config.si.max_sh_per_se = 1;
2944 rdev->config.si.max_backends_per_se = 1;
2945 rdev->config.si.max_texture_channel_caches = 2;
2946 rdev->config.si.max_gprs = 256;
2947 rdev->config.si.max_gs_threads = 16;
2948 rdev->config.si.max_hw_contexts = 8;
2949
2950 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2951 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2952 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2953 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2954 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2955 break;
0a96d72b
AD		 2956 }
2957
2958 /* Initialize HDP */
2959 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2960 WREG32((0x2c14 + j), 0x00000000);
2961 WREG32((0x2c18 + j), 0x00000000);
2962 WREG32((0x2c1c + j), 0x00000000);
2963 WREG32((0x2c20 + j), 0x00000000);
2964 WREG32((0x2c24 + j), 0x00000000);
2965 }
2966
2967 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2968
2969 evergreen_fix_pci_max_read_req_size(rdev);
2970
2971 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2972
2973 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2974 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2975
0a96d72b 2976 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 2977 rdev->config.si.mem_max_burst_length_bytes = 256;
2978 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2979 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2980 if (rdev->config.si.mem_row_size_in_kb > 4)
2981 rdev->config.si.mem_row_size_in_kb = 4;
2982 /* XXX use MC settings? */
2983 rdev->config.si.shader_engine_tile_size = 32;
2984 rdev->config.si.num_gpus = 1;
2985 rdev->config.si.multi_gpu_tile_size = 64;
2986
1a8ca750
AD		 2987 /* fix up row size */
2988 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 2989 switch (rdev->config.si.mem_row_size_in_kb) {
2990 case 1:
2991 default:
2992 gb_addr_config |= ROW_SIZE(0);
2993 break;
2994 case 2:
2995 gb_addr_config |= ROW_SIZE(1);
2996 break;
2997 case 4:
2998 gb_addr_config |= ROW_SIZE(2);
2999 break;
3000 }
3001
0a96d72b
AD		 3002 /* setup tiling info dword. gb_addr_config is not adequate since it does
3003 * not have bank info, so create a custom tiling dword.
3004 * bits 3:0 num_pipes
3005 * bits 7:4 num_banks
3006 * bits 11:8 group_size
3007 * bits 15:12 row_size
3008 */
3009 rdev->config.si.tile_config = 0;
3010 switch (rdev->config.si.num_tile_pipes) {
3011 case 1:
3012 rdev->config.si.tile_config |= (0 << 0);
3013 break;
3014 case 2:
3015 rdev->config.si.tile_config |= (1 << 0);
3016 break;
3017 case 4:
3018 rdev->config.si.tile_config |= (2 << 0);
3019 break;
3020 case 8:
3021 default:
3022 /* XXX what about 12? */
3023 rdev->config.si.tile_config |= (3 << 0);
3024 break;
dca571a6
CK		 3025 }
3026 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3027 case 0: /* four banks */
1a8ca750 3028 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3029 break;
3030 case 1: /* eight banks */
3031 rdev->config.si.tile_config |= 1 << 4;
3032 break;
3033 case 2: /* sixteen banks */
3034 default:
3035 rdev->config.si.tile_config |= 2 << 4;
3036 break;
3037 }
0a96d72b
AD		 3038 rdev->config.si.tile_config |=
3039 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3040 rdev->config.si.tile_config |=
3041 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3042
0a96d72b
AD		 3043 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3044 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3045 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3046 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3047 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3048 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3049 if (rdev->has_uvd) {
3050 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3051 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3052 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3053 }
0a96d72b 3054
1a8ca750 3055 si_tiling_mode_table_init(rdev);
0a96d72b 3056
1a8ca750
AD		 3057 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3058 rdev->config.si.max_sh_per_se,
3059 rdev->config.si.max_backends_per_se);
0a96d72b 3060
1a8ca750
AD		 3061 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3062 rdev->config.si.max_sh_per_se,
3063 rdev->config.si.max_cu_per_sh);
0a96d72b 3064
0a96d72b
AD		 3065
3066 /* set HW defaults for 3D engine */
3067 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3068 ROQ_IB2_START(0x2b)));
3069 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3070
3071 sx_debug_1 = RREG32(SX_DEBUG_1);
3072 WREG32(SX_DEBUG_1, sx_debug_1);
3073
3074 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3075
3076 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3077 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3078 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3079 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3080
3081 WREG32(VGT_NUM_INSTANCES, 1);
3082
3083 WREG32(CP_PERFMON_CNTL, 0);
3084
3085 WREG32(SQ_CONFIG, 0);
3086
3087 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3088 FORCE_EOV_MAX_REZ_CNT(255)));
3089
3090 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3091 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3092
3093 WREG32(VGT_GS_VERTEX_REUSE, 16);
3094 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3095
3096 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3097 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3098 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3099 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3100 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3101 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3102 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3103 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3104
3105 tmp = RREG32(HDP_MISC_CNTL);
3106 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3107 WREG32(HDP_MISC_CNTL, tmp);
3108
3109 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3110 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3111
3112 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3113
3114 udelay(50);
3115}
c476dde2 3116
2ece2e8b
AD		 3117/*
3118 * GPU scratch registers helpers function.
3119 */
3120static void si_scratch_init(struct radeon_device *rdev)
3121{
3122 int i;
3123
3124 rdev->scratch.num_reg = 7;
3125 rdev->scratch.reg_base = SCRATCH_REG0;
3126 for (i = 0; i < rdev->scratch.num_reg; i++) {
3127 rdev->scratch.free[i] = true;
3128 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3129 }
3130}
3131
3132void si_fence_ring_emit(struct radeon_device *rdev,
3133 struct radeon_fence *fence)
3134{
3135 struct radeon_ring *ring = &rdev->ring[fence->ring];
3136 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3137
3138 /* flush read cache over gart */
3139 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3140 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3141 radeon_ring_write(ring, 0);
3142 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3143 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3144 PACKET3_TC_ACTION_ENA |
3145 PACKET3_SH_KCACHE_ACTION_ENA |
3146 PACKET3_SH_ICACHE_ACTION_ENA);
3147 radeon_ring_write(ring, 0xFFFFFFFF);
3148 radeon_ring_write(ring, 0);
3149 radeon_ring_write(ring, 10); /* poll interval */
3150 /* EVENT_WRITE_EOP - flush caches, send int */
3151 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3152 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3153 radeon_ring_write(ring, addr & 0xffffffff);
3154 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3155 radeon_ring_write(ring, fence->seq);
3156 radeon_ring_write(ring, 0);
3157}
3158
3159/*
3160 * IB stuff
3161 */
3162void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3163{
876dc9f3 3164 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 3165 u32 header;
3166
a85a7da4
AD		 3167 if (ib->is_const_ib) {
3168 /* set switch buffer packet before const IB */
3169 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3170 radeon_ring_write(ring, 0);
45df6803 3171
2ece2e8b 3172 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3173 } else {
89d35807 3174 u32 next_rptr;
a85a7da4 3175 if (ring->rptr_save_reg) {
89d35807 3176 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3177 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3178 radeon_ring_write(ring, ((ring->rptr_save_reg -
3179 PACKET3_SET_CONFIG_REG_START) >> 2));
3180 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3181 } else if (rdev->wb.enabled) {
3182 next_rptr = ring->wptr + 5 + 4 + 8;
3183 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3184 radeon_ring_write(ring, (1 << 8));
3185 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3186 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3187 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3188 }
3189
2ece2e8b 3190 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3191 }
2ece2e8b
AD		 3192
3193 radeon_ring_write(ring, header);
3194 radeon_ring_write(ring,
3195#ifdef __BIG_ENDIAN
3196 (2 << 0) |
3197#endif
3198 (ib->gpu_addr & 0xFFFFFFFC));
3199 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 3200 radeon_ring_write(ring, ib->length_dw |
3201 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 3202
a85a7da4
AD		 3203 if (!ib->is_const_ib) {
3204 /* flush read cache over gart for this vmid */
3205 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3206 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 3207 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 3208 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3209 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3210 PACKET3_TC_ACTION_ENA |
3211 PACKET3_SH_KCACHE_ACTION_ENA |
3212 PACKET3_SH_ICACHE_ACTION_ENA);
3213 radeon_ring_write(ring, 0xFFFFFFFF);
3214 radeon_ring_write(ring, 0);
3215 radeon_ring_write(ring, 10); /* poll interval */
3216 }
2ece2e8b
AD		 3217}
3218
48c0c902
AD		 3219/*
3220 * CP.
3221 */
3222static void si_cp_enable(struct radeon_device *rdev, bool enable)
3223{
3224 if (enable)
3225 WREG32(CP_ME_CNTL, 0);
3226 else {
3227 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3228 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3229 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3230 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3231 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3232 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3233 }
3234 udelay(50);
3235}
3236
3237static int si_cp_load_microcode(struct radeon_device *rdev)
3238{
3239 const __be32 *fw_data;
3240 int i;
3241
3242 if (!rdev->me_fw || !rdev->pfp_fw)
3243 return -EINVAL;
3244
3245 si_cp_enable(rdev, false);
3246
3247 /* PFP */
3248 fw_data = (const __be32 *)rdev->pfp_fw->data;
3249 WREG32(CP_PFP_UCODE_ADDR, 0);
3250 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3251 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3252 WREG32(CP_PFP_UCODE_ADDR, 0);
3253
3254 /* CE */
3255 fw_data = (const __be32 *)rdev->ce_fw->data;
3256 WREG32(CP_CE_UCODE_ADDR, 0);
3257 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3258 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3259 WREG32(CP_CE_UCODE_ADDR, 0);
3260
3261 /* ME */
3262 fw_data = (const __be32 *)rdev->me_fw->data;
3263 WREG32(CP_ME_RAM_WADDR, 0);
3264 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3265 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3266 WREG32(CP_ME_RAM_WADDR, 0);
3267
3268 WREG32(CP_PFP_UCODE_ADDR, 0);
3269 WREG32(CP_CE_UCODE_ADDR, 0);
3270 WREG32(CP_ME_RAM_WADDR, 0);
3271 WREG32(CP_ME_RAM_RADDR, 0);
3272 return 0;
3273}
3274
3275static int si_cp_start(struct radeon_device *rdev)
3276{
3277 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3278 int r, i;
3279
3280 r = radeon_ring_lock(rdev, ring, 7 + 4);
3281 if (r) {
3282 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3283 return r;
3284 }
3285 /* init the CP */
3286 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3287 radeon_ring_write(ring, 0x1);
3288 radeon_ring_write(ring, 0x0);
3289 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3290 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3291 radeon_ring_write(ring, 0);
3292 radeon_ring_write(ring, 0);
3293
3294 /* init the CE partitions */
3295 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3296 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3297 radeon_ring_write(ring, 0xc000);
3298 radeon_ring_write(ring, 0xe000);
3299 radeon_ring_unlock_commit(rdev, ring);
3300
3301 si_cp_enable(rdev, true);
3302
3303 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3304 if (r) {
3305 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3306 return r;
3307 }
3308
3309 /* setup clear context state */
3310 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3311 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3312
3313 for (i = 0; i < si_default_size; i++)
3314 radeon_ring_write(ring, si_default_state[i]);
3315
3316 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3317 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3318
3319 /* set clear context state */
3320 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3321 radeon_ring_write(ring, 0);
3322
3323 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3324 radeon_ring_write(ring, 0x00000316);
3325 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3326 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3327
3328 radeon_ring_unlock_commit(rdev, ring);
3329
3330 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3331 ring = &rdev->ring[i];
3332 r = radeon_ring_lock(rdev, ring, 2);
3333
3334 /* clear the compute context state */
3335 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3336 radeon_ring_write(ring, 0);
3337
3338 radeon_ring_unlock_commit(rdev, ring);
3339 }
3340
3341 return 0;
3342}
3343
3344static void si_cp_fini(struct radeon_device *rdev)
3345{
45df6803 3346 struct radeon_ring *ring;
48c0c902 3347 si_cp_enable(rdev, false);
45df6803
CK		 3348
3349 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3350 radeon_ring_fini(rdev, ring);
3351 radeon_scratch_free(rdev, ring->rptr_save_reg);
3352
3353 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3354 radeon_ring_fini(rdev, ring);
3355 radeon_scratch_free(rdev, ring->rptr_save_reg);
3356
3357 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3358 radeon_ring_fini(rdev, ring);
3359 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3360}
3361
3362static int si_cp_resume(struct radeon_device *rdev)
3363{
3364 struct radeon_ring *ring;
3365 u32 tmp;
3366 u32 rb_bufsz;
3367 int r;
3368
3369 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3370 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3371 SOFT_RESET_PA |
3372 SOFT_RESET_VGT |
3373 SOFT_RESET_SPI |
3374 SOFT_RESET_SX));
3375 RREG32(GRBM_SOFT_RESET);
3376 mdelay(15);
3377 WREG32(GRBM_SOFT_RESET, 0);
3378 RREG32(GRBM_SOFT_RESET);
3379
3380 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3381 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3382
3383 /* Set the write pointer delay */
3384 WREG32(CP_RB_WPTR_DELAY, 0);
3385
3386 WREG32(CP_DEBUG, 0);
3387 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3388
3389 /* ring 0 - compute and gfx */
3390 /* Set ring buffer size */
3391 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3392 rb_bufsz = drm_order(ring->ring_size / 8);
3393 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3394#ifdef __BIG_ENDIAN
3395 tmp |= BUF_SWAP_32BIT;
3396#endif
3397 WREG32(CP_RB0_CNTL, tmp);
3398
3399 /* Initialize the ring buffer's read and write pointers */
3400 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3401 ring->wptr = 0;
3402 WREG32(CP_RB0_WPTR, ring->wptr);
3403
48fc7f7e 3404 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3405 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3406 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3407
3408 if (rdev->wb.enabled)
3409 WREG32(SCRATCH_UMSK, 0xff);
3410 else {
3411 tmp |= RB_NO_UPDATE;
3412 WREG32(SCRATCH_UMSK, 0);
3413 }
3414
3415 mdelay(1);
3416 WREG32(CP_RB0_CNTL, tmp);
3417
3418 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3419
3420 ring->rptr = RREG32(CP_RB0_RPTR);
3421
3422 /* ring1 - compute only */
3423 /* Set ring buffer size */
3424 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3425 rb_bufsz = drm_order(ring->ring_size / 8);
3426 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3427#ifdef __BIG_ENDIAN
3428 tmp |= BUF_SWAP_32BIT;
3429#endif
3430 WREG32(CP_RB1_CNTL, tmp);
3431
3432 /* Initialize the ring buffer's read and write pointers */
3433 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3434 ring->wptr = 0;
3435 WREG32(CP_RB1_WPTR, ring->wptr);
3436
48fc7f7e 3437 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3438 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3439 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3440
3441 mdelay(1);
3442 WREG32(CP_RB1_CNTL, tmp);
3443
3444 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3445
3446 ring->rptr = RREG32(CP_RB1_RPTR);
3447
3448 /* ring2 - compute only */
3449 /* Set ring buffer size */
3450 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3451 rb_bufsz = drm_order(ring->ring_size / 8);
3452 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3453#ifdef __BIG_ENDIAN
3454 tmp |= BUF_SWAP_32BIT;
3455#endif
3456 WREG32(CP_RB2_CNTL, tmp);
3457
3458 /* Initialize the ring buffer's read and write pointers */
3459 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3460 ring->wptr = 0;
3461 WREG32(CP_RB2_WPTR, ring->wptr);
3462
48fc7f7e 3463 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3464 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3465 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3466
3467 mdelay(1);
3468 WREG32(CP_RB2_CNTL, tmp);
3469
3470 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3471
3472 ring->rptr = RREG32(CP_RB2_RPTR);
3473
3474 /* start the rings */
3475 si_cp_start(rdev);
3476 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3477 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3478 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3479 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3480 if (r) {
3481 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3482 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3483 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3484 return r;
3485 }
3486 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3487 if (r) {
3488 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3489 }
3490 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3491 if (r) {
3492 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3493 }
3494
3495 return 0;
3496}
3497
014bb209 3498static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3499{
014bb209 3500 u32 reset_mask = 0;
1c534671 3501 u32 tmp;
06bc6df0 3502
014bb209
AD		 3503 /* GRBM_STATUS */
3504 tmp = RREG32(GRBM_STATUS);
3505 if (tmp & (PA_BUSY | SC_BUSY |
3506 BCI_BUSY | SX_BUSY |
3507 TA_BUSY | VGT_BUSY |
3508 DB_BUSY | CB_BUSY |
3509 GDS_BUSY | SPI_BUSY |
3510 IA_BUSY | IA_BUSY_NO_DMA))
3511 reset_mask |= RADEON_RESET_GFX;
3512
3513 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3514 CP_BUSY | CP_COHERENCY_BUSY))
3515 reset_mask |= RADEON_RESET_CP;
3516
3517 if (tmp & GRBM_EE_BUSY)
3518 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3519
3520 /* GRBM_STATUS2 */
3521 tmp = RREG32(GRBM_STATUS2);
3522 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3523 reset_mask |= RADEON_RESET_RLC;
3524
3525 /* DMA_STATUS_REG 0 */
3526 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3527 if (!(tmp & DMA_IDLE))
3528 reset_mask |= RADEON_RESET_DMA;
3529
3530 /* DMA_STATUS_REG 1 */
3531 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3532 if (!(tmp & DMA_IDLE))
3533 reset_mask |= RADEON_RESET_DMA1;
3534
3535 /* SRBM_STATUS2 */
3536 tmp = RREG32(SRBM_STATUS2);
3537 if (tmp & DMA_BUSY)
3538 reset_mask |= RADEON_RESET_DMA;
3539
3540 if (tmp & DMA1_BUSY)
3541 reset_mask |= RADEON_RESET_DMA1;
3542
3543 /* SRBM_STATUS */
3544 tmp = RREG32(SRBM_STATUS);
3545
3546 if (tmp & IH_BUSY)
3547 reset_mask |= RADEON_RESET_IH;
3548
3549 if (tmp & SEM_BUSY)
3550 reset_mask |= RADEON_RESET_SEM;
3551
3552 if (tmp & GRBM_RQ_PENDING)
3553 reset_mask |= RADEON_RESET_GRBM;
3554
3555 if (tmp & VMC_BUSY)
3556 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3557
014bb209
AD		 3558 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3559 MCC_BUSY | MCD_BUSY))
3560 reset_mask |= RADEON_RESET_MC;
3561
3562 if (evergreen_is_display_hung(rdev))
3563 reset_mask |= RADEON_RESET_DISPLAY;
3564
3565 /* VM_L2_STATUS */
3566 tmp = RREG32(VM_L2_STATUS);
3567 if (tmp & L2_BUSY)
3568 reset_mask |= RADEON_RESET_VMC;
3569
d808fc88
AD		 3570 /* Skip MC reset as it's mostly likely not hung, just busy */
3571 if (reset_mask & RADEON_RESET_MC) {
3572 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3573 reset_mask &= ~RADEON_RESET_MC;
3574 }
3575
014bb209
AD		 3576 return reset_mask;
3577}
3578
3579static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3580{
3581 struct evergreen_mc_save save;
3582 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3583 u32 tmp;
19fc42ed 3584
06bc6df0 3585 if (reset_mask == 0)
014bb209 3586 return;
06bc6df0
AD		 3587
3588 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3589
1c534671 3590 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3591 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3592 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3593 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3594 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3595
1c534671
AD		 3596 /* Disable CP parsing/prefetching */
3597 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3598
3599 if (reset_mask & RADEON_RESET_DMA) {
3600 /* dma0 */
3601 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3602 tmp &= ~DMA_RB_ENABLE;
3603 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3604 }
3605 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3606 /* dma1 */
3607 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3608 tmp &= ~DMA_RB_ENABLE;
3609 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3610 }
3611
f770d78a
AD		 3612 udelay(50);
3613
3614 evergreen_mc_stop(rdev, &save);
3615 if (evergreen_mc_wait_for_idle(rdev)) {
3616 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3617 }
3618
1c534671
AD		 3619 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3620 grbm_soft_reset = SOFT_RESET_CB |
3621 SOFT_RESET_DB |
3622 SOFT_RESET_GDS |
3623 SOFT_RESET_PA |
3624 SOFT_RESET_SC |
3625 SOFT_RESET_BCI |
3626 SOFT_RESET_SPI |
3627 SOFT_RESET_SX |
3628 SOFT_RESET_TC |
3629 SOFT_RESET_TA |
3630 SOFT_RESET_VGT |
3631 SOFT_RESET_IA;
3632 }
3633
3634 if (reset_mask & RADEON_RESET_CP) {
3635 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3636
3637 srbm_soft_reset |= SOFT_RESET_GRBM;
3638 }
06bc6df0
AD		 3639
3640 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3641 srbm_soft_reset |= SOFT_RESET_DMA;
3642
3643 if (reset_mask & RADEON_RESET_DMA1)
3644 srbm_soft_reset |= SOFT_RESET_DMA1;
3645
3646 if (reset_mask & RADEON_RESET_DISPLAY)
3647 srbm_soft_reset |= SOFT_RESET_DC;
3648
3649 if (reset_mask & RADEON_RESET_RLC)
3650 grbm_soft_reset |= SOFT_RESET_RLC;
3651
3652 if (reset_mask & RADEON_RESET_SEM)
3653 srbm_soft_reset |= SOFT_RESET_SEM;
3654
3655 if (reset_mask & RADEON_RESET_IH)
3656 srbm_soft_reset |= SOFT_RESET_IH;
3657
3658 if (reset_mask & RADEON_RESET_GRBM)
3659 srbm_soft_reset |= SOFT_RESET_GRBM;
3660
3661 if (reset_mask & RADEON_RESET_VMC)
3662 srbm_soft_reset |= SOFT_RESET_VMC;
3663
3664 if (reset_mask & RADEON_RESET_MC)
3665 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3666
3667 if (grbm_soft_reset) {
3668 tmp = RREG32(GRBM_SOFT_RESET);
3669 tmp |= grbm_soft_reset;
3670 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3671 WREG32(GRBM_SOFT_RESET, tmp);
3672 tmp = RREG32(GRBM_SOFT_RESET);
3673
3674 udelay(50);
3675
3676 tmp &= ~grbm_soft_reset;
3677 WREG32(GRBM_SOFT_RESET, tmp);
3678 tmp = RREG32(GRBM_SOFT_RESET);
3679 }
3680
3681 if (srbm_soft_reset) {
3682 tmp = RREG32(SRBM_SOFT_RESET);
3683 tmp |= srbm_soft_reset;
3684 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3685 WREG32(SRBM_SOFT_RESET, tmp);
3686 tmp = RREG32(SRBM_SOFT_RESET);
3687
3688 udelay(50);
3689
3690 tmp &= ~srbm_soft_reset;
3691 WREG32(SRBM_SOFT_RESET, tmp);
3692 tmp = RREG32(SRBM_SOFT_RESET);
3693 }
06bc6df0
AD		 3694
3695 /* Wait a little for things to settle down */
3696 udelay(50);
3697
c476dde2 3698 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3699 udelay(50);
3700
1c534671 3701 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3702}
3703
3704int si_asic_reset(struct radeon_device *rdev)
3705{
014bb209
AD		 3706 u32 reset_mask;
3707
3708 reset_mask = si_gpu_check_soft_reset(rdev);
3709
3710 if (reset_mask)
3711 r600_set_bios_scratch_engine_hung(rdev, true);
3712
3713 si_gpu_soft_reset(rdev, reset_mask);
3714
3715 reset_mask = si_gpu_check_soft_reset(rdev);
3716
3717 if (!reset_mask)
3718 r600_set_bios_scratch_engine_hung(rdev, false);
3719
3720 return 0;
c476dde2
AD		 3721}
3722
123bc183
AD		 3723/**
3724 * si_gfx_is_lockup - Check if the GFX engine is locked up
3725 *
3726 * @rdev: radeon_device pointer
3727 * @ring: radeon_ring structure holding ring information
3728 *
3729 * Check if the GFX engine is locked up.
3730 * Returns true if the engine appears to be locked up, false if not.
3731 */
3732bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3733{
3734 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3735
3736 if (!(reset_mask & (RADEON_RESET_GFX |
3737 RADEON_RESET_COMPUTE |
3738 RADEON_RESET_CP))) {
3739 radeon_ring_lockup_update(ring);
3740 return false;
3741 }
3742 /* force CP activities */
3743 radeon_ring_force_activity(rdev, ring);
3744 return radeon_ring_test_lockup(rdev, ring);
3745}
3746
3747/**
3748 * si_dma_is_lockup - Check if the DMA engine is locked up
3749 *
3750 * @rdev: radeon_device pointer
3751 * @ring: radeon_ring structure holding ring information
3752 *
3753 * Check if the async DMA engine is locked up.
3754 * Returns true if the engine appears to be locked up, false if not.
3755 */
3756bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3757{
3758 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3759 u32 mask;
3760
3761 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3762 mask = RADEON_RESET_DMA;
3763 else
3764 mask = RADEON_RESET_DMA1;
3765
3766 if (!(reset_mask & mask)) {
3767 radeon_ring_lockup_update(ring);
3768 return false;
3769 }
3770 /* force ring activities */
3771 radeon_ring_force_activity(rdev, ring);
3772 return radeon_ring_test_lockup(rdev, ring);
3773}
3774
d2800ee5
AD		 3775/* MC */
3776static void si_mc_program(struct radeon_device *rdev)
3777{
3778 struct evergreen_mc_save save;
3779 u32 tmp;
3780 int i, j;
3781
3782 /* Initialize HDP */
3783 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3784 WREG32((0x2c14 + j), 0x00000000);
3785 WREG32((0x2c18 + j), 0x00000000);
3786 WREG32((0x2c1c + j), 0x00000000);
3787 WREG32((0x2c20 + j), 0x00000000);
3788 WREG32((0x2c24 + j), 0x00000000);
3789 }
3790 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3791
3792 evergreen_mc_stop(rdev, &save);
3793 if (radeon_mc_wait_for_idle(rdev)) {
3794 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3795 }
5153550a
AD		 3796 if (!ASIC_IS_NODCE(rdev))
3797 /* Lockout access through VGA aperture*/
3798 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3799 /* Update configuration */
3800 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3801 rdev->mc.vram_start >> 12);
3802 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3803 rdev->mc.vram_end >> 12);
3804 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3805 rdev->vram_scratch.gpu_addr >> 12);
3806 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3807 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3808 WREG32(MC_VM_FB_LOCATION, tmp);
3809 /* XXX double check these! */
3810 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3811 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3812 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3813 WREG32(MC_VM_AGP_BASE, 0);
3814 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3815 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3816 if (radeon_mc_wait_for_idle(rdev)) {
3817 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3818 }
3819 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3820 if (!ASIC_IS_NODCE(rdev)) {
3821 /* we need to own VRAM, so turn off the VGA renderer here
3822 * to stop it overwriting our objects */
3823 rv515_vga_render_disable(rdev);
3824 }
d2800ee5
AD		 3825}
3826
1c49165d
AD		 3827void si_vram_gtt_location(struct radeon_device *rdev,
3828 struct radeon_mc *mc)
d2800ee5
AD		 3829{
3830 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3831 /* leave room for at least 1024M GTT */
3832 dev_warn(rdev->dev, "limiting VRAM\n");
3833 mc->real_vram_size = 0xFFC0000000ULL;
3834 mc->mc_vram_size = 0xFFC0000000ULL;
3835 }
9ed8b1f9 3836 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3837 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3838 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3839}
3840
3841static int si_mc_init(struct radeon_device *rdev)
3842{
3843 u32 tmp;
3844 int chansize, numchan;
3845
3846 /* Get VRAM informations */
3847 rdev->mc.vram_is_ddr = true;
3848 tmp = RREG32(MC_ARB_RAMCFG);
3849 if (tmp & CHANSIZE_OVERRIDE) {
3850 chansize = 16;
3851 } else if (tmp & CHANSIZE_MASK) {
3852 chansize = 64;
3853 } else {
3854 chansize = 32;
3855 }
3856 tmp = RREG32(MC_SHARED_CHMAP);
3857 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3858 case 0:
3859 default:
3860 numchan = 1;
3861 break;
3862 case 1:
3863 numchan = 2;
3864 break;
3865 case 2:
3866 numchan = 4;
3867 break;
3868 case 3:
3869 numchan = 8;
3870 break;
3871 case 4:
3872 numchan = 3;
3873 break;
3874 case 5:
3875 numchan = 6;
3876 break;
3877 case 6:
3878 numchan = 10;
3879 break;
3880 case 7:
3881 numchan = 12;
3882 break;
3883 case 8:
3884 numchan = 16;
3885 break;
3886 }
3887 rdev->mc.vram_width = numchan * chansize;
3888 /* Could aper size report 0 ? */
3889 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3890 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3891 /* size in MB on si */
fc986034
NOS		 3892 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3893 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
d2800ee5
AD		 3894 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3895 si_vram_gtt_location(rdev, &rdev->mc);
3896 radeon_update_bandwidth_info(rdev);
3897
3898 return 0;
3899}
3900
3901/*
3902 * GART
3903 */
3904void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3905{
3906 /* flush hdp cache */
3907 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3908
3909 /* bits 0-15 are the VM contexts0-15 */
3910 WREG32(VM_INVALIDATE_REQUEST, 1);
3911}
3912
1109ca09 3913static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 3914{
3915 int r, i;
3916
3917 if (rdev->gart.robj == NULL) {
3918 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3919 return -EINVAL;
3920 }
3921 r = radeon_gart_table_vram_pin(rdev);
3922 if (r)
3923 return r;
3924 radeon_gart_restore(rdev);
3925 /* Setup TLB control */
3926 WREG32(MC_VM_MX_L1_TLB_CNTL,
3927 (0xA << 7) |
3928 ENABLE_L1_TLB |
3929 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3930 ENABLE_ADVANCED_DRIVER_MODEL |
3931 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3932 /* Setup L2 cache */
3933 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3934 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3935 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3936 EFFECTIVE_L2_QUEUE_SIZE(7) |
3937 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3938 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3939 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3940 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3941 /* setup context0 */
3942 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3943 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3944 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3945 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3946 (u32)(rdev->dummy_page.addr >> 12));
3947 WREG32(VM_CONTEXT0_CNTL2, 0);
3948 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3949 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3950
3951 WREG32(0x15D4, 0);
3952 WREG32(0x15D8, 0);
3953 WREG32(0x15DC, 0);
3954
3955 /* empty context1-15 */
d2800ee5
AD		 3956 /* set vm size, must be a multiple of 4 */
3957 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 3958 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 3959 /* Assign the pt base to something valid for now; the pts used for
3960 * the VMs are determined by the application and setup and assigned
3961 * on the fly in the vm part of radeon_gart.c
3962 */
d2800ee5
AD		 3963 for (i = 1; i < 16; i++) {
3964 if (i < 8)
3965 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3966 rdev->gart.table_addr >> 12);
3967 else
3968 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3969 rdev->gart.table_addr >> 12);
3970 }
3971
3972 /* enable context1-15 */
3973 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3974 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 3975 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 3976 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 3977 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3978 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3979 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3980 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3981 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3982 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3983 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3984 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3985 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3986 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3987 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3988 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 3989
3990 si_pcie_gart_tlb_flush(rdev);
3991 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3992 (unsigned)(rdev->mc.gtt_size >> 20),
3993 (unsigned long long)rdev->gart.table_addr);
3994 rdev->gart.ready = true;
3995 return 0;
3996}
3997
1109ca09 3998static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 3999{
4000 /* Disable all tables */
4001 WREG32(VM_CONTEXT0_CNTL, 0);
4002 WREG32(VM_CONTEXT1_CNTL, 0);
4003 /* Setup TLB control */
4004 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4005 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4006 /* Setup L2 cache */
4007 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4008 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4009 EFFECTIVE_L2_QUEUE_SIZE(7) |
4010 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4011 WREG32(VM_L2_CNTL2, 0);
4012 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4013 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4014 radeon_gart_table_vram_unpin(rdev);
4015}
4016
1109ca09 4017static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 4018{
4019 si_pcie_gart_disable(rdev);
4020 radeon_gart_table_vram_free(rdev);
4021 radeon_gart_fini(rdev);
4022}
4023
498dd8b3
AD		 4024/* vm parser */
4025static bool si_vm_reg_valid(u32 reg)
4026{
4027 /* context regs are fine */
4028 if (reg >= 0x28000)
4029 return true;
4030
4031 /* check config regs */
4032 switch (reg) {
4033 case GRBM_GFX_INDEX:
f418b88a 4034 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4035 case VGT_VTX_VECT_EJECT_REG:
4036 case VGT_CACHE_INVALIDATION:
4037 case VGT_ESGS_RING_SIZE:
4038 case VGT_GSVS_RING_SIZE:
4039 case VGT_GS_VERTEX_REUSE:
4040 case VGT_PRIMITIVE_TYPE:
4041 case VGT_INDEX_TYPE:
4042 case VGT_NUM_INDICES:
4043 case VGT_NUM_INSTANCES:
4044 case VGT_TF_RING_SIZE:
4045 case VGT_HS_OFFCHIP_PARAM:
4046 case VGT_TF_MEMORY_BASE:
4047 case PA_CL_ENHANCE:
4048 case PA_SU_LINE_STIPPLE_VALUE:
4049 case PA_SC_LINE_STIPPLE_STATE:
4050 case PA_SC_ENHANCE:
4051 case SQC_CACHES:
4052 case SPI_STATIC_THREAD_MGMT_1:
4053 case SPI_STATIC_THREAD_MGMT_2:
4054 case SPI_STATIC_THREAD_MGMT_3:
4055 case SPI_PS_MAX_WAVE_ID:
4056 case SPI_CONFIG_CNTL:
4057 case SPI_CONFIG_CNTL_1:
4058 case TA_CNTL_AUX:
4059 return true;
4060 default:
4061 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4062 return false;
4063 }
4064}
4065
4066static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4067 u32 *ib, struct radeon_cs_packet *pkt)
4068{
4069 switch (pkt->opcode) {
4070 case PACKET3_NOP:
4071 case PACKET3_SET_BASE:
4072 case PACKET3_SET_CE_DE_COUNTERS:
4073 case PACKET3_LOAD_CONST_RAM:
4074 case PACKET3_WRITE_CONST_RAM:
4075 case PACKET3_WRITE_CONST_RAM_OFFSET:
4076 case PACKET3_DUMP_CONST_RAM:
4077 case PACKET3_INCREMENT_CE_COUNTER:
4078 case PACKET3_WAIT_ON_DE_COUNTER:
4079 case PACKET3_CE_WRITE:
4080 break;
4081 default:
4082 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4083 return -EINVAL;
4084 }
4085 return 0;
4086}
4087
4088static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4089 u32 *ib, struct radeon_cs_packet *pkt)
4090{
4091 u32 idx = pkt->idx + 1;
4092 u32 idx_value = ib[idx];
4093 u32 start_reg, end_reg, reg, i;
5aa709be 4094 u32 command, info;
498dd8b3
AD		 4095
4096 switch (pkt->opcode) {
4097 case PACKET3_NOP:
4098 case PACKET3_SET_BASE:
4099 case PACKET3_CLEAR_STATE:
4100 case PACKET3_INDEX_BUFFER_SIZE:
4101 case PACKET3_DISPATCH_DIRECT:
4102 case PACKET3_DISPATCH_INDIRECT:
4103 case PACKET3_ALLOC_GDS:
4104 case PACKET3_WRITE_GDS_RAM:
4105 case PACKET3_ATOMIC_GDS:
4106 case PACKET3_ATOMIC:
4107 case PACKET3_OCCLUSION_QUERY:
4108 case PACKET3_SET_PREDICATION:
4109 case PACKET3_COND_EXEC:
4110 case PACKET3_PRED_EXEC:
4111 case PACKET3_DRAW_INDIRECT:
4112 case PACKET3_DRAW_INDEX_INDIRECT:
4113 case PACKET3_INDEX_BASE:
4114 case PACKET3_DRAW_INDEX_2:
4115 case PACKET3_CONTEXT_CONTROL:
4116 case PACKET3_INDEX_TYPE:
4117 case PACKET3_DRAW_INDIRECT_MULTI:
4118 case PACKET3_DRAW_INDEX_AUTO:
4119 case PACKET3_DRAW_INDEX_IMMD:
4120 case PACKET3_NUM_INSTANCES:
4121 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4122 case PACKET3_STRMOUT_BUFFER_UPDATE:
4123 case PACKET3_DRAW_INDEX_OFFSET_2:
4124 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4125 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4126 case PACKET3_MPEG_INDEX:
4127 case PACKET3_WAIT_REG_MEM:
4128 case PACKET3_MEM_WRITE:
4129 case PACKET3_PFP_SYNC_ME:
4130 case PACKET3_SURFACE_SYNC:
4131 case PACKET3_EVENT_WRITE:
4132 case PACKET3_EVENT_WRITE_EOP:
4133 case PACKET3_EVENT_WRITE_EOS:
4134 case PACKET3_SET_CONTEXT_REG:
4135 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4136 case PACKET3_SET_SH_REG:
4137 case PACKET3_SET_SH_REG_OFFSET:
4138 case PACKET3_INCREMENT_DE_COUNTER:
4139 case PACKET3_WAIT_ON_CE_COUNTER:
4140 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4141 case PACKET3_ME_WRITE:
4142 break;
4143 case PACKET3_COPY_DATA:
4144 if ((idx_value & 0xf00) == 0) {
4145 reg = ib[idx + 3] * 4;
4146 if (!si_vm_reg_valid(reg))
4147 return -EINVAL;
4148 }
4149 break;
4150 case PACKET3_WRITE_DATA:
4151 if ((idx_value & 0xf00) == 0) {
4152 start_reg = ib[idx + 1] * 4;
4153 if (idx_value & 0x10000) {
4154 if (!si_vm_reg_valid(start_reg))
4155 return -EINVAL;
4156 } else {
4157 for (i = 0; i < (pkt->count - 2); i++) {
4158 reg = start_reg + (4 * i);
4159 if (!si_vm_reg_valid(reg))
4160 return -EINVAL;
4161 }
4162 }
4163 }
4164 break;
4165 case PACKET3_COND_WRITE:
4166 if (idx_value & 0x100) {
4167 reg = ib[idx + 5] * 4;
4168 if (!si_vm_reg_valid(reg))
4169 return -EINVAL;
4170 }
4171 break;
4172 case PACKET3_COPY_DW:
4173 if (idx_value & 0x2) {
4174 reg = ib[idx + 3] * 4;
4175 if (!si_vm_reg_valid(reg))
4176 return -EINVAL;
4177 }
4178 break;
4179 case PACKET3_SET_CONFIG_REG:
4180 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4181 end_reg = 4 * pkt->count + start_reg - 4;
4182 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4183 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4184 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4185 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4186 return -EINVAL;
4187 }
4188 for (i = 0; i < pkt->count; i++) {
4189 reg = start_reg + (4 * i);
4190 if (!si_vm_reg_valid(reg))
4191 return -EINVAL;
4192 }
4193 break;
5aa709be
AD		 4194 case PACKET3_CP_DMA:
4195 command = ib[idx + 4];
4196 info = ib[idx + 1];
4197 if (command & PACKET3_CP_DMA_CMD_SAS) {
4198 /* src address space is register */
4199 if (((info & 0x60000000) >> 29) == 0) {
4200 start_reg = idx_value << 2;
4201 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4202 reg = start_reg;
4203 if (!si_vm_reg_valid(reg)) {
4204 DRM_ERROR("CP DMA Bad SRC register\n");
4205 return -EINVAL;
4206 }
4207 } else {
4208 for (i = 0; i < (command & 0x1fffff); i++) {
4209 reg = start_reg + (4 * i);
4210 if (!si_vm_reg_valid(reg)) {
4211 DRM_ERROR("CP DMA Bad SRC register\n");
4212 return -EINVAL;
4213 }
4214 }
4215 }
4216 }
4217 }
4218 if (command & PACKET3_CP_DMA_CMD_DAS) {
4219 /* dst address space is register */
4220 if (((info & 0x00300000) >> 20) == 0) {
4221 start_reg = ib[idx + 2];
4222 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4223 reg = start_reg;
4224 if (!si_vm_reg_valid(reg)) {
4225 DRM_ERROR("CP DMA Bad DST register\n");
4226 return -EINVAL;
4227 }
4228 } else {
4229 for (i = 0; i < (command & 0x1fffff); i++) {
4230 reg = start_reg + (4 * i);
4231 if (!si_vm_reg_valid(reg)) {
4232 DRM_ERROR("CP DMA Bad DST register\n");
4233 return -EINVAL;
4234 }
4235 }
4236 }
4237 }
4238 }
4239 break;
498dd8b3
AD		 4240 default:
4241 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4242 return -EINVAL;
4243 }
4244 return 0;
4245}
4246
4247static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4248 u32 *ib, struct radeon_cs_packet *pkt)
4249{
4250 u32 idx = pkt->idx + 1;
4251 u32 idx_value = ib[idx];
4252 u32 start_reg, reg, i;
4253
4254 switch (pkt->opcode) {
4255 case PACKET3_NOP:
4256 case PACKET3_SET_BASE:
4257 case PACKET3_CLEAR_STATE:
4258 case PACKET3_DISPATCH_DIRECT:
4259 case PACKET3_DISPATCH_INDIRECT:
4260 case PACKET3_ALLOC_GDS:
4261 case PACKET3_WRITE_GDS_RAM:
4262 case PACKET3_ATOMIC_GDS:
4263 case PACKET3_ATOMIC:
4264 case PACKET3_OCCLUSION_QUERY:
4265 case PACKET3_SET_PREDICATION:
4266 case PACKET3_COND_EXEC:
4267 case PACKET3_PRED_EXEC:
4268 case PACKET3_CONTEXT_CONTROL:
4269 case PACKET3_STRMOUT_BUFFER_UPDATE:
4270 case PACKET3_WAIT_REG_MEM:
4271 case PACKET3_MEM_WRITE:
4272 case PACKET3_PFP_SYNC_ME:
4273 case PACKET3_SURFACE_SYNC:
4274 case PACKET3_EVENT_WRITE:
4275 case PACKET3_EVENT_WRITE_EOP:
4276 case PACKET3_EVENT_WRITE_EOS:
4277 case PACKET3_SET_CONTEXT_REG:
4278 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4279 case PACKET3_SET_SH_REG:
4280 case PACKET3_SET_SH_REG_OFFSET:
4281 case PACKET3_INCREMENT_DE_COUNTER:
4282 case PACKET3_WAIT_ON_CE_COUNTER:
4283 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4284 case PACKET3_ME_WRITE:
4285 break;
4286 case PACKET3_COPY_DATA:
4287 if ((idx_value & 0xf00) == 0) {
4288 reg = ib[idx + 3] * 4;
4289 if (!si_vm_reg_valid(reg))
4290 return -EINVAL;
4291 }
4292 break;
4293 case PACKET3_WRITE_DATA:
4294 if ((idx_value & 0xf00) == 0) {
4295 start_reg = ib[idx + 1] * 4;
4296 if (idx_value & 0x10000) {
4297 if (!si_vm_reg_valid(start_reg))
4298 return -EINVAL;
4299 } else {
4300 for (i = 0; i < (pkt->count - 2); i++) {
4301 reg = start_reg + (4 * i);
4302 if (!si_vm_reg_valid(reg))
4303 return -EINVAL;
4304 }
4305 }
4306 }
4307 break;
4308 case PACKET3_COND_WRITE:
4309 if (idx_value & 0x100) {
4310 reg = ib[idx + 5] * 4;
4311 if (!si_vm_reg_valid(reg))
4312 return -EINVAL;
4313 }
4314 break;
4315 case PACKET3_COPY_DW:
4316 if (idx_value & 0x2) {
4317 reg = ib[idx + 3] * 4;
4318 if (!si_vm_reg_valid(reg))
4319 return -EINVAL;
4320 }
4321 break;
4322 default:
4323 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4324 return -EINVAL;
4325 }
4326 return 0;
4327}
4328
4329int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4330{
4331 int ret = 0;
4332 u32 idx = 0;
4333 struct radeon_cs_packet pkt;
4334
4335 do {
4336 pkt.idx = idx;
4e872ae2
IH		 4337 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4338 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4339 pkt.one_reg_wr = 0;
4340 switch (pkt.type) {
4e872ae2 4341 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4342 dev_err(rdev->dev, "Packet0 not allowed!\n");
4343 ret = -EINVAL;
4344 break;
4e872ae2 4345 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4346 idx += 1;
4347 break;
4e872ae2
IH		 4348 case RADEON_PACKET_TYPE3:
4349 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4350 if (ib->is_const_ib)
4351 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4352 else {
876dc9f3 4353 switch (ib->ring) {
498dd8b3
AD		 4354 case RADEON_RING_TYPE_GFX_INDEX:
4355 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4356 break;
4357 case CAYMAN_RING_TYPE_CP1_INDEX:
4358 case CAYMAN_RING_TYPE_CP2_INDEX:
4359 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4360 break;
4361 default:
876dc9f3 4362 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4363 ret = -EINVAL;
4364 break;
4365 }
4366 }
4367 idx += pkt.count + 2;
4368 break;
4369 default:
4370 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4371 ret = -EINVAL;
4372 break;
4373 }
4374 if (ret)
4375 break;
4376 } while (idx < ib->length_dw);
4377
4378 return ret;
4379}
4380
d2800ee5
AD		 4381/*
4382 * vm
4383 */
4384int si_vm_init(struct radeon_device *rdev)
4385{
4386 /* number of VMs */
4387 rdev->vm_manager.nvm = 16;
4388 /* base offset of vram pages */
4389 rdev->vm_manager.vram_base_offset = 0;
4390
4391 return 0;
4392}
4393
4394void si_vm_fini(struct radeon_device *rdev)
4395{
4396}
4397
fbf6dc7a
AD		 4398/**
4399 * si_vm_decode_fault - print human readable fault info
4400 *
4401 * @rdev: radeon_device pointer
4402 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4403 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4404 *
4405 * Print human readable fault information (SI).
4406 */
4407static void si_vm_decode_fault(struct radeon_device *rdev,
4408 u32 status, u32 addr)
4409{
4410 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4411 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4412 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4413 char *block;
4414
4415 if (rdev->family == CHIP_TAHITI) {
4416 switch (mc_id) {
4417 case 160:
4418 case 144:
4419 case 96:
4420 case 80:
4421 case 224:
4422 case 208:
4423 case 32:
4424 case 16:
4425 block = "CB";
4426 break;
4427 case 161:
4428 case 145:
4429 case 97:
4430 case 81:
4431 case 225:
4432 case 209:
4433 case 33:
4434 case 17:
4435 block = "CB_FMASK";
4436 break;
4437 case 162:
4438 case 146:
4439 case 98:
4440 case 82:
4441 case 226:
4442 case 210:
4443 case 34:
4444 case 18:
4445 block = "CB_CMASK";
4446 break;
4447 case 163:
4448 case 147:
4449 case 99:
4450 case 83:
4451 case 227:
4452 case 211:
4453 case 35:
4454 case 19:
4455 block = "CB_IMMED";
4456 break;
4457 case 164:
4458 case 148:
4459 case 100:
4460 case 84:
4461 case 228:
4462 case 212:
4463 case 36:
4464 case 20:
4465 block = "DB";
4466 break;
4467 case 165:
4468 case 149:
4469 case 101:
4470 case 85:
4471 case 229:
4472 case 213:
4473 case 37:
4474 case 21:
4475 block = "DB_HTILE";
4476 break;
4477 case 167:
4478 case 151:
4479 case 103:
4480 case 87:
4481 case 231:
4482 case 215:
4483 case 39:
4484 case 23:
4485 block = "DB_STEN";
4486 break;
4487 case 72:
4488 case 68:
4489 case 64:
4490 case 8:
4491 case 4:
4492 case 0:
4493 case 136:
4494 case 132:
4495 case 128:
4496 case 200:
4497 case 196:
4498 case 192:
4499 block = "TC";
4500 break;
4501 case 112:
4502 case 48:
4503 block = "CP";
4504 break;
4505 case 49:
4506 case 177:
4507 case 50:
4508 case 178:
4509 block = "SH";
4510 break;
4511 case 53:
4512 case 190:
4513 block = "VGT";
4514 break;
4515 case 117:
4516 block = "IH";
4517 break;
4518 case 51:
4519 case 115:
4520 block = "RLC";
4521 break;
4522 case 119:
4523 case 183:
4524 block = "DMA0";
4525 break;
4526 case 61:
4527 block = "DMA1";
4528 break;
4529 case 248:
4530 case 120:
4531 block = "HDP";
4532 break;
4533 default:
4534 block = "unknown";
4535 break;
4536 }
4537 } else {
4538 switch (mc_id) {
4539 case 32:
4540 case 16:
4541 case 96:
4542 case 80:
4543 case 160:
4544 case 144:
4545 case 224:
4546 case 208:
4547 block = "CB";
4548 break;
4549 case 33:
4550 case 17:
4551 case 97:
4552 case 81:
4553 case 161:
4554 case 145:
4555 case 225:
4556 case 209:
4557 block = "CB_FMASK";
4558 break;
4559 case 34:
4560 case 18:
4561 case 98:
4562 case 82:
4563 case 162:
4564 case 146:
4565 case 226:
4566 case 210:
4567 block = "CB_CMASK";
4568 break;
4569 case 35:
4570 case 19:
4571 case 99:
4572 case 83:
4573 case 163:
4574 case 147:
4575 case 227:
4576 case 211:
4577 block = "CB_IMMED";
4578 break;
4579 case 36:
4580 case 20:
4581 case 100:
4582 case 84:
4583 case 164:
4584 case 148:
4585 case 228:
4586 case 212:
4587 block = "DB";
4588 break;
4589 case 37:
4590 case 21:
4591 case 101:
4592 case 85:
4593 case 165:
4594 case 149:
4595 case 229:
4596 case 213:
4597 block = "DB_HTILE";
4598 break;
4599 case 39:
4600 case 23:
4601 case 103:
4602 case 87:
4603 case 167:
4604 case 151:
4605 case 231:
4606 case 215:
4607 block = "DB_STEN";
4608 break;
4609 case 72:
4610 case 68:
4611 case 8:
4612 case 4:
4613 case 136:
4614 case 132:
4615 case 200:
4616 case 196:
4617 block = "TC";
4618 break;
4619 case 112:
4620 case 48:
4621 block = "CP";
4622 break;
4623 case 49:
4624 case 177:
4625 case 50:
4626 case 178:
4627 block = "SH";
4628 break;
4629 case 53:
4630 block = "VGT";
4631 break;
4632 case 117:
4633 block = "IH";
4634 break;
4635 case 51:
4636 case 115:
4637 block = "RLC";
4638 break;
4639 case 119:
4640 case 183:
4641 block = "DMA0";
4642 break;
4643 case 61:
4644 block = "DMA1";
4645 break;
4646 case 248:
4647 case 120:
4648 block = "HDP";
4649 break;
4650 default:
4651 block = "unknown";
4652 break;
4653 }
4654 }
4655
4656 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4657 protections, vmid, addr,
4658 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4659 block, mc_id);
4660}
4661
82ffd92b
AD		 4662/**
4663 * si_vm_set_page - update the page tables using the CP
4664 *
4665 * @rdev: radeon_device pointer
43f1214a 4666 * @ib: indirect buffer to fill with commands
82ffd92b
AD		 4667 * @pe: addr of the page entry
4668 * @addr: dst addr to write into pe
4669 * @count: number of page entries to update
4670 * @incr: increase next addr by incr bytes
4671 * @flags: access flags
4672 *
43f1214a 4673 * Update the page tables using the CP (SI).
82ffd92b 4674 */
43f1214a
AD		 4675void si_vm_set_page(struct radeon_device *rdev,
4676 struct radeon_ib *ib,
4677 uint64_t pe,
82ffd92b
AD		 4678 uint64_t addr, unsigned count,
4679 uint32_t incr, uint32_t flags)
d2800ee5 4680{
82ffd92b 4681 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
deab48f1
AD		 4682 uint64_t value;
4683 unsigned ndw;
4684
4685 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4686 while (count) {
4687 ndw = 2 + count * 2;
4688 if (ndw > 0x3FFE)
4689 ndw = 0x3FFE;
4690
43f1214a
AD		 4691 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4692 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4693 WRITE_DATA_DST_SEL(1));
4694 ib->ptr[ib->length_dw++] = pe;
4695 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
deab48f1
AD		 4696 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4697 if (flags & RADEON_VM_PAGE_SYSTEM) {
4698 value = radeon_vm_map_gart(rdev, addr);
4699 value &= 0xFFFFFFFFFFFFF000ULL;
4700 } else if (flags & RADEON_VM_PAGE_VALID) {
4701 value = addr;
4702 } else {
4703 value = 0;
4704 }
4705 addr += incr;
4706 value |= r600_flags;
43f1214a
AD		 4707 ib->ptr[ib->length_dw++] = value;
4708 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4709 }
4710 }
4711 } else {
4712 /* DMA */
4713 if (flags & RADEON_VM_PAGE_SYSTEM) {
4714 while (count) {
4715 ndw = count * 2;
4716 if (ndw > 0xFFFFE)
4717 ndw = 0xFFFFE;
4718
4719 /* for non-physically contiguous pages (system) */
43f1214a
AD		 4720 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
4721 ib->ptr[ib->length_dw++] = pe;
4722 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
deab48f1
AD		 4723 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
4724 if (flags & RADEON_VM_PAGE_SYSTEM) {
4725 value = radeon_vm_map_gart(rdev, addr);
4726 value &= 0xFFFFFFFFFFFFF000ULL;
4727 } else if (flags & RADEON_VM_PAGE_VALID) {
4728 value = addr;
4729 } else {
4730 value = 0;
4731 }
4732 addr += incr;
4733 value |= r600_flags;
43f1214a
AD		 4734 ib->ptr[ib->length_dw++] = value;
4735 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4736 }
4737 }
4738 } else {
4739 while (count) {
4740 ndw = count * 2;
4741 if (ndw > 0xFFFFE)
4742 ndw = 0xFFFFE;
4743
4744 if (flags & RADEON_VM_PAGE_VALID)
4745 value = addr;
4746 else
4747 value = 0;
4748 /* for physically contiguous pages (vram) */
43f1214a
AD		 4749 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
4750 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4751 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4752 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4753 ib->ptr[ib->length_dw++] = 0;
4754 ib->ptr[ib->length_dw++] = value; /* value */
4755 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4756 ib->ptr[ib->length_dw++] = incr; /* increment size */
4757 ib->ptr[ib->length_dw++] = 0;
deab48f1
AD		 4758 pe += ndw * 4;
4759 addr += (ndw / 2) * incr;
4760 count -= ndw / 2;
4761 }
d7025d89 4762 }
43f1214a
AD		 4763 while (ib->length_dw & 0x7)
4764 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
82ffd92b 4765 }
d2800ee5
AD		 4766}
4767
498522b4 4768void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4769{
498522b4 4770 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4771
ee60e29f 4772 if (vm == NULL)
d2800ee5
AD		 4773 return;
4774
76c44f2c
AD		 4775 /* write new base address */
4776 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4777 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4778 WRITE_DATA_DST_SEL(0)));
4779
ee60e29f 4780 if (vm->id < 8) {
76c44f2c
AD		 4781 radeon_ring_write(ring,
4782 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4783 } else {
76c44f2c
AD		 4784 radeon_ring_write(ring,
4785 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4786 }
76c44f2c 4787 radeon_ring_write(ring, 0);
fa87e62d 4788 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4789
d2800ee5 4790 /* flush hdp cache */
76c44f2c
AD		 4791 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4792 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4793 WRITE_DATA_DST_SEL(0)));
4794 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4795 radeon_ring_write(ring, 0);
ee60e29f
CK		 4796 radeon_ring_write(ring, 0x1);
4797
d2800ee5 4798 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4799 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4800 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4801 WRITE_DATA_DST_SEL(0)));
4802 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4803 radeon_ring_write(ring, 0);
498522b4 4804 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4805
4806 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4807 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4808 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4809}
4810
8c5fd7ef
AD		 4811void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4812{
4813 struct radeon_ring *ring = &rdev->ring[ridx];
4814
4815 if (vm == NULL)
4816 return;
4817
4818 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4819 if (vm->id < 8) {
4820 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4821 } else {
4822 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4823 }
4824 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4825
4826 /* flush hdp cache */
4827 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4828 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4829 radeon_ring_write(ring, 1);
4830
4831 /* bits 0-7 are the VM contexts0-7 */
4832 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4833 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4834 radeon_ring_write(ring, 1 << vm->id);
4835}
4836
f8f84ac5
AD		 4837/*
4838 * Power and clock gating
4839 */
4840static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4841{
4842 int i;
4843
4844 for (i = 0; i < rdev->usec_timeout; i++) {
4845 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4846 break;
4847 udelay(1);
4848 }
4849
4850 for (i = 0; i < rdev->usec_timeout; i++) {
4851 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4852 break;
4853 udelay(1);
4854 }
4855}
4856
4857static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4858 bool enable)
4859{
4860 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4861 u32 mask;
4862 int i;
4863
4864 if (enable)
4865 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4866 else
4867 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4868 WREG32(CP_INT_CNTL_RING0, tmp);
4869
4870 if (!enable) {
4871 /* read a gfx register */
4872 tmp = RREG32(DB_DEPTH_INFO);
4873
4874 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4875 for (i = 0; i < rdev->usec_timeout; i++) {
4876 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4877 break;
4878 udelay(1);
4879 }
4880 }
4881}
4882
4883static void si_set_uvd_dcm(struct radeon_device *rdev,
4884 bool sw_mode)
4885{
4886 u32 tmp, tmp2;
4887
4888 tmp = RREG32(UVD_CGC_CTRL);
4889 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4890 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4891
4892 if (sw_mode) {
4893 tmp &= ~0x7ffff800;
4894 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4895 } else {
4896 tmp |= 0x7ffff800;
4897 tmp2 = 0;
4898 }
4899
4900 WREG32(UVD_CGC_CTRL, tmp);
4901 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4902}
4903
4904static void si_init_uvd_internal_cg(struct radeon_device *rdev)
4905{
4906 bool hw_mode = true;
4907
4908 if (hw_mode) {
4909 si_set_uvd_dcm(rdev, false);
4910 } else {
4911 u32 tmp = RREG32(UVD_CGC_CTRL);
4912 tmp &= ~DCM;
4913 WREG32(UVD_CGC_CTRL, tmp);
4914 }
4915}
4916
4917static u32 si_halt_rlc(struct radeon_device *rdev)
4918{
4919 u32 data, orig;
4920
4921 orig = data = RREG32(RLC_CNTL);
4922
4923 if (data & RLC_ENABLE) {
4924 data &= ~RLC_ENABLE;
4925 WREG32(RLC_CNTL, data);
4926
4927 si_wait_for_rlc_serdes(rdev);
4928 }
4929
4930 return orig;
4931}
4932
4933static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4934{
4935 u32 tmp;
4936
4937 tmp = RREG32(RLC_CNTL);
4938 if (tmp != rlc)
4939 WREG32(RLC_CNTL, rlc);
4940}
4941
4942static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4943{
4944 u32 data, orig;
4945
4946 orig = data = RREG32(DMA_PG);
4947 if (enable)
4948 data |= PG_CNTL_ENABLE;
4949 else
4950 data &= ~PG_CNTL_ENABLE;
4951 if (orig != data)
4952 WREG32(DMA_PG, data);
4953}
4954
4955static void si_init_dma_pg(struct radeon_device *rdev)
4956{
4957 u32 tmp;
4958
4959 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4960 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4961
4962 for (tmp = 0; tmp < 5; tmp++)
4963 WREG32(DMA_PGFSM_WRITE, 0);
4964}
4965
4966static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4967 bool enable)
4968{
4969 u32 tmp;
4970
4971 if (enable) {
4972 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4973 WREG32(RLC_TTOP_D, tmp);
4974
4975 tmp = RREG32(RLC_PG_CNTL);
4976 tmp |= GFX_PG_ENABLE;
4977 WREG32(RLC_PG_CNTL, tmp);
4978
4979 tmp = RREG32(RLC_AUTO_PG_CTRL);
4980 tmp |= AUTO_PG_EN;
4981 WREG32(RLC_AUTO_PG_CTRL, tmp);
4982 } else {
4983 tmp = RREG32(RLC_AUTO_PG_CTRL);
4984 tmp &= ~AUTO_PG_EN;
4985 WREG32(RLC_AUTO_PG_CTRL, tmp);
4986
4987 tmp = RREG32(DB_RENDER_CONTROL);
4988 }
4989}
4990
4991static void si_init_gfx_cgpg(struct radeon_device *rdev)
4992{
4993 u32 tmp;
4994
4995 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4996
4997 tmp = RREG32(RLC_PG_CNTL);
4998 tmp |= GFX_PG_SRC;
4999 WREG32(RLC_PG_CNTL, tmp);
5000
5001 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5002
5003 tmp = RREG32(RLC_AUTO_PG_CTRL);
5004
5005 tmp &= ~GRBM_REG_SGIT_MASK;
5006 tmp |= GRBM_REG_SGIT(0x700);
5007 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5008 WREG32(RLC_AUTO_PG_CTRL, tmp);
5009}
5010
ba19031a 5011static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
f8f84ac5
AD		 5012{
5013 u32 mask = 0, tmp, tmp1;
5014 int i;
5015
5016 si_select_se_sh(rdev, se, sh);
5017 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5018 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5019 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5020
5021 tmp &= 0xffff0000;
5022
5023 tmp |= tmp1;
5024 tmp >>= 16;
5025
5026 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5027 mask <<= 1;
5028 mask |= 1;
5029 }
5030
5031 return (~tmp) & mask;
5032}
5033
5034static void si_init_ao_cu_mask(struct radeon_device *rdev)
5035{
5036 u32 i, j, k, active_cu_number = 0;
5037 u32 mask, counter, cu_bitmap;
5038 u32 tmp = 0;
5039
5040 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5041 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5042 mask = 1;
5043 cu_bitmap = 0;
5044 counter = 0;
5045 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
ba19031a 5046 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
f8f84ac5
AD		 5047 if (counter < 2)
5048 cu_bitmap |= mask;
5049 counter++;
5050 }
5051 mask <<= 1;
5052 }
5053
5054 active_cu_number += counter;
5055 tmp |= (cu_bitmap << (i * 16 + j * 8));
5056 }
5057 }
5058
5059 WREG32(RLC_PG_AO_CU_MASK, tmp);
5060
5061 tmp = RREG32(RLC_MAX_PG_CU);
5062 tmp &= ~MAX_PU_CU_MASK;
5063 tmp |= MAX_PU_CU(active_cu_number);
5064 WREG32(RLC_MAX_PG_CU, tmp);
5065}
5066
5067static void si_enable_cgcg(struct radeon_device *rdev,
5068 bool enable)
5069{
5070 u32 data, orig, tmp;
5071
5072 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5073
5074 si_enable_gui_idle_interrupt(rdev, enable);
5075
5076 if (enable) {
5077 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5078
5079 tmp = si_halt_rlc(rdev);
5080
5081 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5082 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5083 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5084
5085 si_wait_for_rlc_serdes(rdev);
5086
5087 si_update_rlc(rdev, tmp);
5088
5089 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5090
5091 data |= CGCG_EN | CGLS_EN;
5092 } else {
5093 RREG32(CB_CGTT_SCLK_CTRL);
5094 RREG32(CB_CGTT_SCLK_CTRL);
5095 RREG32(CB_CGTT_SCLK_CTRL);
5096 RREG32(CB_CGTT_SCLK_CTRL);
5097
5098 data &= ~(CGCG_EN | CGLS_EN);
5099 }
5100
5101 if (orig != data)
5102 WREG32(RLC_CGCG_CGLS_CTRL, data);
5103}
5104
5105static void si_enable_mgcg(struct radeon_device *rdev,
5106 bool enable)
5107{
5108 u32 data, orig, tmp = 0;
5109
5110 if (enable) {
5111 orig = data = RREG32(CGTS_SM_CTRL_REG);
5112 data = 0x96940200;
5113 if (orig != data)
5114 WREG32(CGTS_SM_CTRL_REG, data);
5115
5116 orig = data = RREG32(CP_MEM_SLP_CNTL);
5117 data |= CP_MEM_LS_EN;
5118 if (orig != data)
5119 WREG32(CP_MEM_SLP_CNTL, data);
5120
5121 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5122 data &= 0xffffffc0;
5123 if (orig != data)
5124 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5125
5126 tmp = si_halt_rlc(rdev);
5127
5128 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5129 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5130 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5131
5132 si_update_rlc(rdev, tmp);
5133 } else {
5134 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5135 data |= 0x00000003;
5136 if (orig != data)
5137 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5138
5139 data = RREG32(CP_MEM_SLP_CNTL);
5140 if (data & CP_MEM_LS_EN) {
5141 data &= ~CP_MEM_LS_EN;
5142 WREG32(CP_MEM_SLP_CNTL, data);
5143 }
5144 orig = data = RREG32(CGTS_SM_CTRL_REG);
5145 data |= LS_OVERRIDE | OVERRIDE;
5146 if (orig != data)
5147 WREG32(CGTS_SM_CTRL_REG, data);
5148
5149 tmp = si_halt_rlc(rdev);
5150
5151 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5152 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5153 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5154
5155 si_update_rlc(rdev, tmp);
5156 }
5157}
5158
5159static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5160 bool enable)
5161{
5162 u32 orig, data, tmp;
5163
5164 if (enable) {
5165 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5166 tmp |= 0x3fff;
5167 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5168
5169 orig = data = RREG32(UVD_CGC_CTRL);
5170 data |= DCM;
5171 if (orig != data)
5172 WREG32(UVD_CGC_CTRL, data);
5173
5174 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5175 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5176 } else {
5177 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5178 tmp &= ~0x3fff;
5179 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5180
5181 orig = data = RREG32(UVD_CGC_CTRL);
5182 data &= ~DCM;
5183 if (orig != data)
5184 WREG32(UVD_CGC_CTRL, data);
5185
5186 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5187 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5188 }
5189}
5190
5191static const u32 mc_cg_registers[] =
5192{
5193 MC_HUB_MISC_HUB_CG,
5194 MC_HUB_MISC_SIP_CG,
5195 MC_HUB_MISC_VM_CG,
5196 MC_XPB_CLK_GAT,
5197 ATC_MISC_CG,
5198 MC_CITF_MISC_WR_CG,
5199 MC_CITF_MISC_RD_CG,
5200 MC_CITF_MISC_VM_CG,
5201 VM_L2_CG,
5202};
5203
5204static void si_enable_mc_ls(struct radeon_device *rdev,
5205 bool enable)
5206{
5207 int i;
5208 u32 orig, data;
5209
5210 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5211 orig = data = RREG32(mc_cg_registers[i]);
5212 if (enable)
5213 data |= MC_LS_ENABLE;
5214 else
5215 data &= ~MC_LS_ENABLE;
5216 if (data != orig)
5217 WREG32(mc_cg_registers[i], data);
5218 }
5219}
5220
5221
5222static void si_init_cg(struct radeon_device *rdev)
5223{
f8f84ac5 5224 si_enable_mgcg(rdev, true);
3652f005 5225 si_enable_cgcg(rdev, false);
f8f84ac5
AD		 5226 /* disable MC LS on Tahiti */
5227 if (rdev->family == CHIP_TAHITI)
5228 si_enable_mc_ls(rdev, false);
b2d70917 5229 if (rdev->has_uvd) {
f8f84ac5
AD		 5230 si_enable_uvd_mgcg(rdev, true);
5231 si_init_uvd_internal_cg(rdev);
5232 }
5233}
5234
5235static void si_fini_cg(struct radeon_device *rdev)
5236{
b2d70917 5237 if (rdev->has_uvd)
f8f84ac5
AD		 5238 si_enable_uvd_mgcg(rdev, false);
5239 si_enable_cgcg(rdev, false);
5240 si_enable_mgcg(rdev, false);
5241}
5242
5243static void si_init_pg(struct radeon_device *rdev)
5244{
5245 bool has_pg = false;
3652f005 5246#if 0
f8f84ac5
AD		 5247 /* only cape verde supports PG */
5248 if (rdev->family == CHIP_VERDE)
5249 has_pg = true;
3652f005 5250#endif
f8f84ac5
AD		 5251 if (has_pg) {
5252 si_init_ao_cu_mask(rdev);
5253 si_init_dma_pg(rdev);
5254 si_enable_dma_pg(rdev, true);
5255 si_init_gfx_cgpg(rdev);
5256 si_enable_gfx_cgpg(rdev, true);
5257 } else {
5258 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5259 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5260 }
5261}
5262
5263static void si_fini_pg(struct radeon_device *rdev)
5264{
5265 bool has_pg = false;
5266
5267 /* only cape verde supports PG */
5268 if (rdev->family == CHIP_VERDE)
5269 has_pg = true;
5270
5271 if (has_pg) {
5272 si_enable_dma_pg(rdev, false);
5273 si_enable_gfx_cgpg(rdev, false);
5274 }
5275}
5276
347e7592
AD		 5277/*
5278 * RLC
5279 */
866d83de 5280void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5281{
f8f84ac5 5282 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5283
f8f84ac5
AD		 5284 tmp |= SOFT_RESET_RLC;
5285 WREG32(GRBM_SOFT_RESET, tmp);
5286 udelay(50);
5287 tmp &= ~SOFT_RESET_RLC;
5288 WREG32(GRBM_SOFT_RESET, tmp);
5289 udelay(50);
d719cef3
AD		 5290}
5291
347e7592
AD		 5292static void si_rlc_stop(struct radeon_device *rdev)
5293{
5294 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5295
5296 si_enable_gui_idle_interrupt(rdev, false);
5297
5298 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5299}
5300
5301static void si_rlc_start(struct radeon_device *rdev)
5302{
5303 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5304
5305 si_enable_gui_idle_interrupt(rdev, true);
5306
5307 udelay(50);
5308}
5309
5310static bool si_lbpw_supported(struct radeon_device *rdev)
5311{
5312 u32 tmp;
5313
5314 /* Enable LBPW only for DDR3 */
5315 tmp = RREG32(MC_SEQ_MISC0);
5316 if ((tmp & 0xF0000000) == 0xB0000000)
5317 return true;
5318 return false;
5319}
5320
5321static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5322{
5323 u32 tmp;
5324
5325 tmp = RREG32(RLC_LB_CNTL);
5326 if (enable)
5327 tmp |= LOAD_BALANCE_ENABLE;
5328 else
5329 tmp &= ~LOAD_BALANCE_ENABLE;
5330 WREG32(RLC_LB_CNTL, tmp);
5331
5332 if (!enable) {
5333 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5334 WREG32(SPI_LB_CU_MASK, 0x00ff);
5335 }
347e7592
AD		 5336}
5337
5338static int si_rlc_resume(struct radeon_device *rdev)
5339{
5340 u32 i;
5341 const __be32 *fw_data;
5342
5343 if (!rdev->rlc_fw)
5344 return -EINVAL;
5345
5346 si_rlc_stop(rdev);
5347
f8f84ac5
AD		 5348 si_rlc_reset(rdev);
5349
5350 si_init_pg(rdev);
5351
5352 si_init_cg(rdev);
5353
347e7592
AD		 5354 WREG32(RLC_RL_BASE, 0);
5355 WREG32(RLC_RL_SIZE, 0);
5356 WREG32(RLC_LB_CNTL, 0);
5357 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5358 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5359 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5360
347e7592
AD		 5361 WREG32(RLC_MC_CNTL, 0);
5362 WREG32(RLC_UCODE_CNTL, 0);
5363
5364 fw_data = (const __be32 *)rdev->rlc_fw->data;
5365 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5366 WREG32(RLC_UCODE_ADDR, i);
5367 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5368 }
5369 WREG32(RLC_UCODE_ADDR, 0);
5370
d719cef3
AD		 5371 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5372
347e7592
AD		 5373 si_rlc_start(rdev);
5374
5375 return 0;
5376}
5377
25a857fb
AD		 5378static void si_enable_interrupts(struct radeon_device *rdev)
5379{
5380 u32 ih_cntl = RREG32(IH_CNTL);
5381 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5382
5383 ih_cntl |= ENABLE_INTR;
5384 ih_rb_cntl |= IH_RB_ENABLE;
5385 WREG32(IH_CNTL, ih_cntl);
5386 WREG32(IH_RB_CNTL, ih_rb_cntl);
5387 rdev->ih.enabled = true;
5388}
5389
5390static void si_disable_interrupts(struct radeon_device *rdev)
5391{
5392 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5393 u32 ih_cntl = RREG32(IH_CNTL);
5394
5395 ih_rb_cntl &= ~IH_RB_ENABLE;
5396 ih_cntl &= ~ENABLE_INTR;
5397 WREG32(IH_RB_CNTL, ih_rb_cntl);
5398 WREG32(IH_CNTL, ih_cntl);
5399 /* set rptr, wptr to 0 */
5400 WREG32(IH_RB_RPTR, 0);
5401 WREG32(IH_RB_WPTR, 0);
5402 rdev->ih.enabled = false;
25a857fb
AD		 5403 rdev->ih.rptr = 0;
5404}
5405
5406static void si_disable_interrupt_state(struct radeon_device *rdev)
5407{
5408 u32 tmp;
5409
5410 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5411 WREG32(CP_INT_CNTL_RING1, 0);
5412 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5413 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5414 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5415 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5416 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5417 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 5418 if (rdev->num_crtc >= 2) {
5419 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5420 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5421 }
25a857fb
AD		 5422 if (rdev->num_crtc >= 4) {
5423 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5424 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5425 }
5426 if (rdev->num_crtc >= 6) {
5427 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5428 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5429 }
5430
5153550a
AD		 5431 if (rdev->num_crtc >= 2) {
5432 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5433 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5434 }
25a857fb
AD		 5435 if (rdev->num_crtc >= 4) {
5436 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5437 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5438 }
5439 if (rdev->num_crtc >= 6) {
5440 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5441 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5442 }
5443
5153550a
AD		 5444 if (!ASIC_IS_NODCE(rdev)) {
5445 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5446
5447 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5448 WREG32(DC_HPD1_INT_CONTROL, tmp);
5449 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5450 WREG32(DC_HPD2_INT_CONTROL, tmp);
5451 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5452 WREG32(DC_HPD3_INT_CONTROL, tmp);
5453 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5454 WREG32(DC_HPD4_INT_CONTROL, tmp);
5455 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5456 WREG32(DC_HPD5_INT_CONTROL, tmp);
5457 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5458 WREG32(DC_HPD6_INT_CONTROL, tmp);
5459 }
25a857fb
AD		 5460}
5461
5462static int si_irq_init(struct radeon_device *rdev)
5463{
5464 int ret = 0;
5465 int rb_bufsz;
5466 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5467
5468 /* allocate ring */
5469 ret = r600_ih_ring_alloc(rdev);
5470 if (ret)
5471 return ret;
5472
5473 /* disable irqs */
5474 si_disable_interrupts(rdev);
5475
5476 /* init rlc */
5477 ret = si_rlc_resume(rdev);
5478 if (ret) {
5479 r600_ih_ring_fini(rdev);
5480 return ret;
5481 }
5482
5483 /* setup interrupt control */
5484 /* set dummy read address to ring address */
5485 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5486 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5487 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5488 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5489 */
5490 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5491 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5492 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5493 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5494
5495 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5496 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
5497
5498 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5499 IH_WPTR_OVERFLOW_CLEAR |
5500 (rb_bufsz << 1));
5501
5502 if (rdev->wb.enabled)
5503 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5504
5505 /* set the writeback address whether it's enabled or not */
5506 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5507 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5508
5509 WREG32(IH_RB_CNTL, ih_rb_cntl);
5510
5511 /* set rptr, wptr to 0 */
5512 WREG32(IH_RB_RPTR, 0);
5513 WREG32(IH_RB_WPTR, 0);
5514
5515 /* Default settings for IH_CNTL (disabled at first) */
5516 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5517 /* RPTR_REARM only works if msi's are enabled */
5518 if (rdev->msi_enabled)
5519 ih_cntl |= RPTR_REARM;
5520 WREG32(IH_CNTL, ih_cntl);
5521
5522 /* force the active interrupt state to all disabled */
5523 si_disable_interrupt_state(rdev);
5524
2099810f
DA		 5525 pci_set_master(rdev->pdev);
5526
25a857fb
AD		 5527 /* enable irqs */
5528 si_enable_interrupts(rdev);
5529
5530 return ret;
5531}
5532
5533int si_irq_set(struct radeon_device *rdev)
5534{
5535 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
5536 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5537 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 5538 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 5539 u32 grbm_int_cntl = 0;
5540 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 5541 u32 dma_cntl, dma_cntl1;
a9e61410 5542 u32 thermal_int = 0;
25a857fb
AD		 5543
5544 if (!rdev->irq.installed) {
5545 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5546 return -EINVAL;
5547 }
5548 /* don't enable anything if the ih is disabled */
5549 if (!rdev->ih.enabled) {
5550 si_disable_interrupts(rdev);
5551 /* force the active interrupt state to all disabled */
5552 si_disable_interrupt_state(rdev);
5553 return 0;
5554 }
5555
5153550a
AD		 5556 if (!ASIC_IS_NODCE(rdev)) {
5557 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5558 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5559 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5560 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5561 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5562 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5563 }
25a857fb 5564
8c5fd7ef
AD		 5565 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5566 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5567
a9e61410
AD		 5568 thermal_int = RREG32(CG_THERMAL_INT) &
5569 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5570
25a857fb 5571 /* enable CP interrupts on all rings */
736fc37f 5572 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 5573 DRM_DEBUG("si_irq_set: sw int gfx\n");
5574 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5575 }
736fc37f 5576 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 5577 DRM_DEBUG("si_irq_set: sw int cp1\n");
5578 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5579 }
736fc37f 5580 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 5581 DRM_DEBUG("si_irq_set: sw int cp2\n");
5582 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5583 }
8c5fd7ef
AD		 5584 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5585 DRM_DEBUG("si_irq_set: sw int dma\n");
5586 dma_cntl |= TRAP_ENABLE;
5587 }
5588
5589 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5590 DRM_DEBUG("si_irq_set: sw int dma1\n");
5591 dma_cntl1 |= TRAP_ENABLE;
5592 }
25a857fb 5593 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 5594 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 5595 DRM_DEBUG("si_irq_set: vblank 0\n");
5596 crtc1 |= VBLANK_INT_MASK;
5597 }
5598 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 5599 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 5600 DRM_DEBUG("si_irq_set: vblank 1\n");
5601 crtc2 |= VBLANK_INT_MASK;
5602 }
5603 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 5604 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 5605 DRM_DEBUG("si_irq_set: vblank 2\n");
5606 crtc3 |= VBLANK_INT_MASK;
5607 }
5608 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 5609 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 5610 DRM_DEBUG("si_irq_set: vblank 3\n");
5611 crtc4 |= VBLANK_INT_MASK;
5612 }
5613 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 5614 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 5615 DRM_DEBUG("si_irq_set: vblank 4\n");
5616 crtc5 |= VBLANK_INT_MASK;
5617 }
5618 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 5619 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 5620 DRM_DEBUG("si_irq_set: vblank 5\n");
5621 crtc6 |= VBLANK_INT_MASK;
5622 }
5623 if (rdev->irq.hpd[0]) {
5624 DRM_DEBUG("si_irq_set: hpd 1\n");
5625 hpd1 |= DC_HPDx_INT_EN;
5626 }
5627 if (rdev->irq.hpd[1]) {
5628 DRM_DEBUG("si_irq_set: hpd 2\n");
5629 hpd2 |= DC_HPDx_INT_EN;
5630 }
5631 if (rdev->irq.hpd[2]) {
5632 DRM_DEBUG("si_irq_set: hpd 3\n");
5633 hpd3 |= DC_HPDx_INT_EN;
5634 }
5635 if (rdev->irq.hpd[3]) {
5636 DRM_DEBUG("si_irq_set: hpd 4\n");
5637 hpd4 |= DC_HPDx_INT_EN;
5638 }
5639 if (rdev->irq.hpd[4]) {
5640 DRM_DEBUG("si_irq_set: hpd 5\n");
5641 hpd5 |= DC_HPDx_INT_EN;
5642 }
5643 if (rdev->irq.hpd[5]) {
5644 DRM_DEBUG("si_irq_set: hpd 6\n");
5645 hpd6 |= DC_HPDx_INT_EN;
5646 }
25a857fb
AD		 5647
5648 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5649 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5650 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5651
8c5fd7ef
AD		 5652 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5653 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5654
25a857fb
AD		 5655 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5656
a9e61410
AD		 5657 if (rdev->irq.dpm_thermal) {
5658 DRM_DEBUG("dpm thermal\n");
5659 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5660 }
5661
5153550a
AD		 5662 if (rdev->num_crtc >= 2) {
5663 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5664 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5665 }
25a857fb
AD		 5666 if (rdev->num_crtc >= 4) {
5667 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5668 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5669 }
5670 if (rdev->num_crtc >= 6) {
5671 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5672 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5673 }
5674
5153550a
AD		 5675 if (rdev->num_crtc >= 2) {
5676 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5677 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5678 }
25a857fb
AD		 5679 if (rdev->num_crtc >= 4) {
5680 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5681 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5682 }
5683 if (rdev->num_crtc >= 6) {
5684 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5685 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5686 }
5687
5153550a
AD		 5688 if (!ASIC_IS_NODCE(rdev)) {
5689 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5690 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5691 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5692 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5693 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5694 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5695 }
25a857fb 5696
a9e61410
AD		 5697 WREG32(CG_THERMAL_INT, thermal_int);
5698
25a857fb
AD		 5699 return 0;
5700}
5701
5702static inline void si_irq_ack(struct radeon_device *rdev)
5703{
5704 u32 tmp;
5705
5153550a
AD		 5706 if (ASIC_IS_NODCE(rdev))
5707 return;
5708
25a857fb
AD		 5709 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5710 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5711 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5712 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5713 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5714 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5715 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5716 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5717 if (rdev->num_crtc >= 4) {
5718 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5719 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5720 }
5721 if (rdev->num_crtc >= 6) {
5722 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5723 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5724 }
5725
5726 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5727 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5728 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5729 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5730 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5731 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5732 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5733 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5734 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5735 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5736 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5737 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5738
5739 if (rdev->num_crtc >= 4) {
5740 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5741 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5742 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5743 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5744 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5745 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5746 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5747 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5748 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5749 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5750 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5751 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5752 }
5753
5754 if (rdev->num_crtc >= 6) {
5755 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5756 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5757 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5758 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5759 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5760 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5761 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5762 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5763 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5764 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5765 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5766 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5767 }
5768
5769 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5770 tmp = RREG32(DC_HPD1_INT_CONTROL);
5771 tmp |= DC_HPDx_INT_ACK;
5772 WREG32(DC_HPD1_INT_CONTROL, tmp);
5773 }
5774 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5775 tmp = RREG32(DC_HPD2_INT_CONTROL);
5776 tmp |= DC_HPDx_INT_ACK;
5777 WREG32(DC_HPD2_INT_CONTROL, tmp);
5778 }
5779 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5780 tmp = RREG32(DC_HPD3_INT_CONTROL);
5781 tmp |= DC_HPDx_INT_ACK;
5782 WREG32(DC_HPD3_INT_CONTROL, tmp);
5783 }
5784 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5785 tmp = RREG32(DC_HPD4_INT_CONTROL);
5786 tmp |= DC_HPDx_INT_ACK;
5787 WREG32(DC_HPD4_INT_CONTROL, tmp);
5788 }
5789 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5790 tmp = RREG32(DC_HPD5_INT_CONTROL);
5791 tmp |= DC_HPDx_INT_ACK;
5792 WREG32(DC_HPD5_INT_CONTROL, tmp);
5793 }
5794 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5795 tmp = RREG32(DC_HPD5_INT_CONTROL);
5796 tmp |= DC_HPDx_INT_ACK;
5797 WREG32(DC_HPD6_INT_CONTROL, tmp);
5798 }
5799}
5800
5801static void si_irq_disable(struct radeon_device *rdev)
5802{
5803 si_disable_interrupts(rdev);
5804 /* Wait and acknowledge irq */
5805 mdelay(1);
5806 si_irq_ack(rdev);
5807 si_disable_interrupt_state(rdev);
5808}
5809
5810static void si_irq_suspend(struct radeon_device *rdev)
5811{
5812 si_irq_disable(rdev);
5813 si_rlc_stop(rdev);
5814}
5815
9b136d51
AD		 5816static void si_irq_fini(struct radeon_device *rdev)
5817{
5818 si_irq_suspend(rdev);
5819 r600_ih_ring_fini(rdev);
5820}
5821
25a857fb
AD		 5822static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
5823{
5824 u32 wptr, tmp;
5825
5826 if (rdev->wb.enabled)
5827 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5828 else
5829 wptr = RREG32(IH_RB_WPTR);
5830
5831 if (wptr & RB_OVERFLOW) {
5832 /* When a ring buffer overflow happen start parsing interrupt
5833 * from the last not overwritten vector (wptr + 16). Hopefully
5834 * this should allow us to catchup.
5835 */
5836 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5837 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
5838 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5839 tmp = RREG32(IH_RB_CNTL);
5840 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5841 WREG32(IH_RB_CNTL, tmp);
5842 }
5843 return (wptr & rdev->ih.ptr_mask);
5844}
5845
5846/* SI IV Ring
5847 * Each IV ring entry is 128 bits:
5848 * [7:0] - interrupt source id
5849 * [31:8] - reserved
5850 * [59:32] - interrupt source data
5851 * [63:60] - reserved
5852 * [71:64] - RINGID
5853 * [79:72] - VMID
5854 * [127:80] - reserved
5855 */
5856int si_irq_process(struct radeon_device *rdev)
5857{
5858 u32 wptr;
5859 u32 rptr;
5860 u32 src_id, src_data, ring_id;
5861 u32 ring_index;
25a857fb 5862 bool queue_hotplug = false;
a9e61410 5863 bool queue_thermal = false;
fbf6dc7a 5864 u32 status, addr;
25a857fb
AD		 5865
5866 if (!rdev->ih.enabled || rdev->shutdown)
5867 return IRQ_NONE;
5868
5869 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 5870
5871restart_ih:
5872 /* is somebody else already processing irqs? */
5873 if (atomic_xchg(&rdev->ih.lock, 1))
5874 return IRQ_NONE;
5875
25a857fb
AD		 5876 rptr = rdev->ih.rptr;
5877 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
5878
25a857fb
AD		 5879 /* Order reading of wptr vs. reading of IH ring data */
5880 rmb();
5881
5882 /* display interrupts */
5883 si_irq_ack(rdev);
5884
25a857fb
AD		 5885 while (rptr != wptr) {
5886 /* wptr/rptr are in bytes! */
5887 ring_index = rptr / 4;
5888 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
5889 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
5890 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
5891
5892 switch (src_id) {
5893 case 1: /* D1 vblank/vline */
5894 switch (src_data) {
5895 case 0: /* D1 vblank */
5896 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
5897 if (rdev->irq.crtc_vblank_int[0]) {
5898 drm_handle_vblank(rdev->ddev, 0);
5899 rdev->pm.vblank_sync = true;
5900 wake_up(&rdev->irq.vblank_queue);
5901 }
736fc37f 5902 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 5903 radeon_crtc_handle_flip(rdev, 0);
5904 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
5905 DRM_DEBUG("IH: D1 vblank\n");
5906 }
5907 break;
5908 case 1: /* D1 vline */
5909 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
5910 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
5911 DRM_DEBUG("IH: D1 vline\n");
5912 }
5913 break;
5914 default:
5915 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5916 break;
5917 }
5918 break;
5919 case 2: /* D2 vblank/vline */
5920 switch (src_data) {
5921 case 0: /* D2 vblank */
5922 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
5923 if (rdev->irq.crtc_vblank_int[1]) {
5924 drm_handle_vblank(rdev->ddev, 1);
5925 rdev->pm.vblank_sync = true;
5926 wake_up(&rdev->irq.vblank_queue);
5927 }
736fc37f 5928 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 5929 radeon_crtc_handle_flip(rdev, 1);
5930 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5931 DRM_DEBUG("IH: D2 vblank\n");
5932 }
5933 break;
5934 case 1: /* D2 vline */
5935 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
5936 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5937 DRM_DEBUG("IH: D2 vline\n");
5938 }
5939 break;
5940 default:
5941 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5942 break;
5943 }
5944 break;
5945 case 3: /* D3 vblank/vline */
5946 switch (src_data) {
5947 case 0: /* D3 vblank */
5948 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
5949 if (rdev->irq.crtc_vblank_int[2]) {
5950 drm_handle_vblank(rdev->ddev, 2);
5951 rdev->pm.vblank_sync = true;
5952 wake_up(&rdev->irq.vblank_queue);
5953 }
736fc37f 5954 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 5955 radeon_crtc_handle_flip(rdev, 2);
5956 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5957 DRM_DEBUG("IH: D3 vblank\n");
5958 }
5959 break;
5960 case 1: /* D3 vline */
5961 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
5962 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5963 DRM_DEBUG("IH: D3 vline\n");
5964 }
5965 break;
5966 default:
5967 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5968 break;
5969 }
5970 break;
5971 case 4: /* D4 vblank/vline */
5972 switch (src_data) {
5973 case 0: /* D4 vblank */
5974 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
5975 if (rdev->irq.crtc_vblank_int[3]) {
5976 drm_handle_vblank(rdev->ddev, 3);
5977 rdev->pm.vblank_sync = true;
5978 wake_up(&rdev->irq.vblank_queue);
5979 }
736fc37f 5980 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 5981 radeon_crtc_handle_flip(rdev, 3);
5982 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5983 DRM_DEBUG("IH: D4 vblank\n");
5984 }
5985 break;
5986 case 1: /* D4 vline */
5987 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
5988 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5989 DRM_DEBUG("IH: D4 vline\n");
5990 }
5991 break;
5992 default:
5993 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5994 break;
5995 }
5996 break;
5997 case 5: /* D5 vblank/vline */
5998 switch (src_data) {
5999 case 0: /* D5 vblank */
6000 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6001 if (rdev->irq.crtc_vblank_int[4]) {
6002 drm_handle_vblank(rdev->ddev, 4);
6003 rdev->pm.vblank_sync = true;
6004 wake_up(&rdev->irq.vblank_queue);
6005 }
736fc37f 6006 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 6007 radeon_crtc_handle_flip(rdev, 4);
6008 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6009 DRM_DEBUG("IH: D5 vblank\n");
6010 }
6011 break;
6012 case 1: /* D5 vline */
6013 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6014 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6015 DRM_DEBUG("IH: D5 vline\n");
6016 }
6017 break;
6018 default:
6019 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6020 break;
6021 }
6022 break;
6023 case 6: /* D6 vblank/vline */
6024 switch (src_data) {
6025 case 0: /* D6 vblank */
6026 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6027 if (rdev->irq.crtc_vblank_int[5]) {
6028 drm_handle_vblank(rdev->ddev, 5);
6029 rdev->pm.vblank_sync = true;
6030 wake_up(&rdev->irq.vblank_queue);
6031 }
736fc37f 6032 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 6033 radeon_crtc_handle_flip(rdev, 5);
6034 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6035 DRM_DEBUG("IH: D6 vblank\n");
6036 }
6037 break;
6038 case 1: /* D6 vline */
6039 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6040 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6041 DRM_DEBUG("IH: D6 vline\n");
6042 }
6043 break;
6044 default:
6045 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6046 break;
6047 }
6048 break;
6049 case 42: /* HPD hotplug */
6050 switch (src_data) {
6051 case 0:
6052 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6053 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6054 queue_hotplug = true;
6055 DRM_DEBUG("IH: HPD1\n");
6056 }
6057 break;
6058 case 1:
6059 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6060 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6061 queue_hotplug = true;
6062 DRM_DEBUG("IH: HPD2\n");
6063 }
6064 break;
6065 case 2:
6066 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6067 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6068 queue_hotplug = true;
6069 DRM_DEBUG("IH: HPD3\n");
6070 }
6071 break;
6072 case 3:
6073 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6074 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6075 queue_hotplug = true;
6076 DRM_DEBUG("IH: HPD4\n");
6077 }
6078 break;
6079 case 4:
6080 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6081 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6082 queue_hotplug = true;
6083 DRM_DEBUG("IH: HPD5\n");
6084 }
6085 break;
6086 case 5:
6087 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6088 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6089 queue_hotplug = true;
6090 DRM_DEBUG("IH: HPD6\n");
6091 }
6092 break;
6093 default:
6094 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6095 break;
6096 }
6097 break;
ae133a11
CK		 6098 case 146:
6099 case 147:
fbf6dc7a
AD		 6100 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6101 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 6102 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6103 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
fbf6dc7a 6104 addr);
ae133a11 6105 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
fbf6dc7a
AD		 6106 status);
6107 si_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 6108 /* reset addr and status */
6109 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6110 break;
25a857fb
AD		 6111 case 176: /* RINGID0 CP_INT */
6112 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6113 break;
6114 case 177: /* RINGID1 CP_INT */
6115 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6116 break;
6117 case 178: /* RINGID2 CP_INT */
6118 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6119 break;
6120 case 181: /* CP EOP event */
6121 DRM_DEBUG("IH: CP EOP\n");
6122 switch (ring_id) {
6123 case 0:
6124 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6125 break;
6126 case 1:
6127 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6128 break;
6129 case 2:
6130 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6131 break;
6132 }
6133 break;
8c5fd7ef
AD		 6134 case 224: /* DMA trap event */
6135 DRM_DEBUG("IH: DMA trap\n");
6136 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6137 break;
a9e61410
AD		 6138 case 230: /* thermal low to high */
6139 DRM_DEBUG("IH: thermal low to high\n");
6140 rdev->pm.dpm.thermal.high_to_low = false;
6141 queue_thermal = true;
6142 break;
6143 case 231: /* thermal high to low */
6144 DRM_DEBUG("IH: thermal high to low\n");
6145 rdev->pm.dpm.thermal.high_to_low = true;
6146 queue_thermal = true;
6147 break;
25a857fb
AD		 6148 case 233: /* GUI IDLE */
6149 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6150 break;
8c5fd7ef
AD		 6151 case 244: /* DMA trap event */
6152 DRM_DEBUG("IH: DMA1 trap\n");
6153 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6154 break;
25a857fb
AD		 6155 default:
6156 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6157 break;
6158 }
6159
6160 /* wptr/rptr are in bytes! */
6161 rptr += 16;
6162 rptr &= rdev->ih.ptr_mask;
6163 }
25a857fb
AD		 6164 if (queue_hotplug)
6165 schedule_work(&rdev->hotplug_work);
a9e61410
AD		 6166 if (queue_thermal && rdev->pm.dpm_enabled)
6167 schedule_work(&rdev->pm.dpm.thermal.work);
25a857fb
AD		 6168 rdev->ih.rptr = rptr;
6169 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 6170 atomic_set(&rdev->ih.lock, 0);
6171
6172 /* make sure wptr hasn't changed while processing */
6173 wptr = si_get_ih_wptr(rdev);
6174 if (wptr != rptr)
6175 goto restart_ih;
6176
25a857fb
AD		 6177 return IRQ_HANDLED;
6178}
6179
8c5fd7ef
AD		 6180/**
6181 * si_copy_dma - copy pages using the DMA engine
6182 *
6183 * @rdev: radeon_device pointer
6184 * @src_offset: src GPU address
6185 * @dst_offset: dst GPU address
6186 * @num_gpu_pages: number of GPU pages to xfer
6187 * @fence: radeon fence object
6188 *
6189 * Copy GPU paging using the DMA engine (SI).
6190 * Used by the radeon ttm implementation to move pages if
6191 * registered as the asic copy callback.
6192 */
6193int si_copy_dma(struct radeon_device *rdev,
6194 uint64_t src_offset, uint64_t dst_offset,
6195 unsigned num_gpu_pages,
6196 struct radeon_fence **fence)
6197{
6198 struct radeon_semaphore *sem = NULL;
6199 int ring_index = rdev->asic->copy.dma_ring_index;
6200 struct radeon_ring *ring = &rdev->ring[ring_index];
6201 u32 size_in_bytes, cur_size_in_bytes;
6202 int i, num_loops;
6203 int r = 0;
6204
6205 r = radeon_semaphore_create(rdev, &sem);
6206 if (r) {
6207 DRM_ERROR("radeon: moving bo (%d).\n", r);
6208 return r;
6209 }
6210
6211 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
6212 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
6213 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
6214 if (r) {
6215 DRM_ERROR("radeon: moving bo (%d).\n", r);
6216 radeon_semaphore_free(rdev, &sem, NULL);
6217 return r;
6218 }
6219
6220 if (radeon_fence_need_sync(*fence, ring->idx)) {
6221 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
6222 ring->idx);
6223 radeon_fence_note_sync(*fence, ring->idx);
6224 } else {
6225 radeon_semaphore_free(rdev, &sem, NULL);
6226 }
6227
6228 for (i = 0; i < num_loops; i++) {
6229 cur_size_in_bytes = size_in_bytes;
6230 if (cur_size_in_bytes > 0xFFFFF)
6231 cur_size_in_bytes = 0xFFFFF;
6232 size_in_bytes -= cur_size_in_bytes;
6233 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
6234 radeon_ring_write(ring, dst_offset & 0xffffffff);
6235 radeon_ring_write(ring, src_offset & 0xffffffff);
6236 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
6237 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
6238 src_offset += cur_size_in_bytes;
6239 dst_offset += cur_size_in_bytes;
6240 }
6241
6242 r = radeon_fence_emit(rdev, fence, ring->idx);
6243 if (r) {
6244 radeon_ring_unlock_undo(rdev, ring);
6245 return r;
6246 }
6247
6248 radeon_ring_unlock_commit(rdev, ring);
6249 radeon_semaphore_free(rdev, &sem, *fence);
6250
6251 return r;
6252}
6253
9b136d51
AD		 6254/*
6255 * startup/shutdown callbacks
6256 */
6257static int si_startup(struct radeon_device *rdev)
6258{
6259 struct radeon_ring *ring;
6260 int r;
6261
b9d305df
AD		 6262 /* enable pcie gen2/3 link */
6263 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6264 /* enable aspm */
6265 si_program_aspm(rdev);
b9d305df 6266
6fab3feb
AD		 6267 si_mc_program(rdev);
6268
9b136d51
AD		 6269 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6270 !rdev->rlc_fw || !rdev->mc_fw) {
6271 r = si_init_microcode(rdev);
6272 if (r) {
6273 DRM_ERROR("Failed to load firmware!\n");
6274 return r;
6275 }
6276 }
6277
6278 r = si_mc_load_microcode(rdev);
6279 if (r) {
6280 DRM_ERROR("Failed to load MC firmware!\n");
6281 return r;
6282 }
6283
6284 r = r600_vram_scratch_init(rdev);
6285 if (r)
6286 return r;
6287
9b136d51
AD		 6288 r = si_pcie_gart_enable(rdev);
6289 if (r)
6290 return r;
6291 si_gpu_init(rdev);
6292
9b136d51 6293 /* allocate rlc buffers */
1fd11777
AD		 6294 if (rdev->family == CHIP_VERDE) {
6295 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6296 rdev->rlc.reg_list_size =
6297 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6298 }
6299 rdev->rlc.cs_data = si_cs_data;
6300 r = sumo_rlc_init(rdev);
9b136d51
AD		 6301 if (r) {
6302 DRM_ERROR("Failed to init rlc BOs!\n");
6303 return r;
6304 }
6305
6306 /* allocate wb buffer */
6307 r = radeon_wb_init(rdev);
6308 if (r)
6309 return r;
6310
6311 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6312 if (r) {
6313 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6314 return r;
6315 }
6316
6317 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6318 if (r) {
6319 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6320 return r;
6321 }
6322
6323 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6324 if (r) {
6325 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6326 return r;
6327 }
6328
8c5fd7ef
AD		 6329 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6330 if (r) {
6331 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6332 return r;
6333 }
6334
6335 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6336 if (r) {
6337 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6338 return r;
6339 }
6340
1df0d523
AD		 6341 if (rdev->has_uvd) {
6342 r = rv770_uvd_resume(rdev);
6343 if (!r) {
6344 r = radeon_fence_driver_start_ring(rdev,
6345 R600_RING_TYPE_UVD_INDEX);
6346 if (r)
6347 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6348 }
f2ba57b5 6349 if (r)
1df0d523 6350 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 6351 }
f2ba57b5 6352
9b136d51 6353 /* Enable IRQ */
e49f3959
AH		 6354 if (!rdev->irq.installed) {
6355 r = radeon_irq_kms_init(rdev);
6356 if (r)
6357 return r;
6358 }
6359
9b136d51
AD		 6360 r = si_irq_init(rdev);
6361 if (r) {
6362 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6363 radeon_irq_kms_fini(rdev);
6364 return r;
6365 }
6366 si_irq_set(rdev);
6367
6368 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6369 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6370 CP_RB0_RPTR, CP_RB0_WPTR,
6371 0, 0xfffff, RADEON_CP_PACKET2);
6372 if (r)
6373 return r;
6374
6375 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6376 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6377 CP_RB1_RPTR, CP_RB1_WPTR,
6378 0, 0xfffff, RADEON_CP_PACKET2);
6379 if (r)
6380 return r;
6381
6382 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6383 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6384 CP_RB2_RPTR, CP_RB2_WPTR,
6385 0, 0xfffff, RADEON_CP_PACKET2);
6386 if (r)
6387 return r;
6388
8c5fd7ef
AD		 6389 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6390 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6391 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6392 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
6393 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6394 if (r)
6395 return r;
6396
6397 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6398 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6399 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6400 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
6401 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6402 if (r)
6403 return r;
6404
9b136d51
AD		 6405 r = si_cp_load_microcode(rdev);
6406 if (r)
6407 return r;
6408 r = si_cp_resume(rdev);
6409 if (r)
6410 return r;
6411
8c5fd7ef
AD		 6412 r = cayman_dma_resume(rdev);
6413 if (r)
6414 return r;
6415
1df0d523
AD		 6416 if (rdev->has_uvd) {
6417 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6418 if (ring->ring_size) {
6419 r = radeon_ring_init(rdev, ring, ring->ring_size,
6420 R600_WB_UVD_RPTR_OFFSET,
6421 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6422 0, 0xfffff, RADEON_CP_PACKET2);
6423 if (!r)
6424 r = r600_uvd_init(rdev);
6425 if (r)
6426 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6427 }
f2ba57b5
CK		 6428 }
6429
2898c348
CK		 6430 r = radeon_ib_pool_init(rdev);
6431 if (r) {
6432 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6433 return r;
2898c348 6434 }
9b136d51 6435
c6105f24
CK		 6436 r = radeon_vm_manager_init(rdev);
6437 if (r) {
6438 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6439 return r;
c6105f24 6440 }
9b136d51
AD		 6441
6442 return 0;
6443}
6444
6445int si_resume(struct radeon_device *rdev)
6446{
6447 int r;
6448
6449 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6450 * posting will perform necessary task to bring back GPU into good
6451 * shape.
6452 */
6453 /* post card */
6454 atom_asic_init(rdev->mode_info.atom_context);
6455
205996c0
AD		 6456 /* init golden registers */
6457 si_init_golden_registers(rdev);
6458
9b136d51
AD		 6459 rdev->accel_working = true;
6460 r = si_startup(rdev);
6461 if (r) {
6462 DRM_ERROR("si startup failed on resume\n");
6463 rdev->accel_working = false;
6464 return r;
6465 }
6466
6467 return r;
6468
6469}
6470
6471int si_suspend(struct radeon_device *rdev)
6472{
fa3daf9a 6473 radeon_vm_manager_fini(rdev);
9b136d51 6474 si_cp_enable(rdev, false);
8c5fd7ef 6475 cayman_dma_stop(rdev);
1df0d523 6476 if (rdev->has_uvd) {
2858c00d 6477 r600_uvd_stop(rdev);
1df0d523
AD		 6478 radeon_uvd_suspend(rdev);
6479 }
9b136d51
AD		 6480 si_irq_suspend(rdev);
6481 radeon_wb_disable(rdev);
6482 si_pcie_gart_disable(rdev);
6483 return 0;
6484}
6485
6486/* Plan is to move initialization in that function and use
6487 * helper function so that radeon_device_init pretty much
6488 * do nothing more than calling asic specific function. This
6489 * should also allow to remove a bunch of callback function
6490 * like vram_info.
6491 */
6492int si_init(struct radeon_device *rdev)
6493{
6494 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6495 int r;
6496
9b136d51
AD		 6497 /* Read BIOS */
6498 if (!radeon_get_bios(rdev)) {
6499 if (ASIC_IS_AVIVO(rdev))
6500 return -EINVAL;
6501 }
6502 /* Must be an ATOMBIOS */
6503 if (!rdev->is_atom_bios) {
6504 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6505 return -EINVAL;
6506 }
6507 r = radeon_atombios_init(rdev);
6508 if (r)
6509 return r;
6510
6511 /* Post card if necessary */
6512 if (!radeon_card_posted(rdev)) {
6513 if (!rdev->bios) {
6514 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6515 return -EINVAL;
6516 }
6517 DRM_INFO("GPU not posted. posting now...\n");
6518 atom_asic_init(rdev->mode_info.atom_context);
6519 }
205996c0
AD		 6520 /* init golden registers */
6521 si_init_golden_registers(rdev);
9b136d51
AD		 6522 /* Initialize scratch registers */
6523 si_scratch_init(rdev);
6524 /* Initialize surface registers */
6525 radeon_surface_init(rdev);
6526 /* Initialize clocks */
6527 radeon_get_clock_info(rdev->ddev);
6528
6529 /* Fence driver */
6530 r = radeon_fence_driver_init(rdev);
6531 if (r)
6532 return r;
6533
6534 /* initialize memory controller */
6535 r = si_mc_init(rdev);
6536 if (r)
6537 return r;
6538 /* Memory manager */
6539 r = radeon_bo_init(rdev);
6540 if (r)
6541 return r;
6542
9b136d51
AD		 6543 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6544 ring->ring_obj = NULL;
6545 r600_ring_init(rdev, ring, 1024 * 1024);
6546
6547 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6548 ring->ring_obj = NULL;
6549 r600_ring_init(rdev, ring, 1024 * 1024);
6550
6551 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6552 ring->ring_obj = NULL;
6553 r600_ring_init(rdev, ring, 1024 * 1024);
6554
8c5fd7ef
AD		 6555 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6556 ring->ring_obj = NULL;
6557 r600_ring_init(rdev, ring, 64 * 1024);
6558
6559 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6560 ring->ring_obj = NULL;
6561 r600_ring_init(rdev, ring, 64 * 1024);
6562
1df0d523
AD		 6563 if (rdev->has_uvd) {
6564 r = radeon_uvd_init(rdev);
6565 if (!r) {
6566 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6567 ring->ring_obj = NULL;
6568 r600_ring_init(rdev, ring, 4096);
6569 }
f2ba57b5
CK		 6570 }
6571
9b136d51
AD		 6572 rdev->ih.ring_obj = NULL;
6573 r600_ih_ring_init(rdev, 64 * 1024);
6574
6575 r = r600_pcie_gart_init(rdev);
6576 if (r)
6577 return r;
6578
9b136d51 6579 rdev->accel_working = true;
9b136d51
AD		 6580 r = si_startup(rdev);
6581 if (r) {
6582 dev_err(rdev->dev, "disabling GPU acceleration\n");
6583 si_cp_fini(rdev);
8c5fd7ef 6584 cayman_dma_fini(rdev);
9b136d51 6585 si_irq_fini(rdev);
1fd11777 6586 sumo_rlc_fini(rdev);
9b136d51 6587 radeon_wb_fini(rdev);
2898c348 6588 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6589 radeon_vm_manager_fini(rdev);
6590 radeon_irq_kms_fini(rdev);
6591 si_pcie_gart_fini(rdev);
6592 rdev->accel_working = false;
6593 }
6594
6595 /* Don't start up if the MC ucode is missing.
6596 * The default clocks and voltages before the MC ucode
6597 * is loaded are not suffient for advanced operations.
6598 */
6599 if (!rdev->mc_fw) {
6600 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6601 return -EINVAL;
6602 }
6603
6604 return 0;
6605}
6606
6607void si_fini(struct radeon_device *rdev)
6608{
9b136d51 6609 si_cp_fini(rdev);
8c5fd7ef 6610 cayman_dma_fini(rdev);
9b136d51 6611 si_irq_fini(rdev);
1fd11777 6612 sumo_rlc_fini(rdev);
f8f84ac5
AD		 6613 si_fini_cg(rdev);
6614 si_fini_pg(rdev);
9b136d51
AD		 6615 radeon_wb_fini(rdev);
6616 radeon_vm_manager_fini(rdev);
2898c348 6617 radeon_ib_pool_fini(rdev);
9b136d51 6618 radeon_irq_kms_fini(rdev);
2858c00d
CK		 6619 if (rdev->has_uvd) {
6620 r600_uvd_stop(rdev);
1df0d523 6621 radeon_uvd_fini(rdev);
2858c00d 6622 }
9b136d51
AD		 6623 si_pcie_gart_fini(rdev);
6624 r600_vram_scratch_fini(rdev);
6625 radeon_gem_fini(rdev);
9b136d51
AD		 6626 radeon_fence_driver_fini(rdev);
6627 radeon_bo_fini(rdev);
6628 radeon_atombios_fini(rdev);
6629 kfree(rdev->bios);
6630 rdev->bios = NULL;
6631}
6632
6759a0a7 6633/**
d0418894 6634 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6635 *
6636 * @rdev: radeon_device pointer
6637 *
6638 * Fetches a GPU clock counter snapshot (SI).
6639 * Returns the 64 bit clock counter snapshot.
6640 */
d0418894 6641uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6642{
6643 uint64_t clock;
6644
6645 mutex_lock(&rdev->gpu_clock_mutex);
6646 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6647 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6648 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6649 mutex_unlock(&rdev->gpu_clock_mutex);
6650 return clock;
6651}
2539eb02 6652
2539eb02
CK		 6653int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6654{
facd112d 6655 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6656 int r;
6657
4ed10835
CK		 6658 /* bypass vclk and dclk with bclk */
6659 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6660 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6661 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6662
6663 /* put PLL in bypass mode */
6664 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6665
6666 if (!vclk || !dclk) {
6667 /* keep the Bypass mode, put PLL to sleep */
6668 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6669 return 0;
6670 }
6671
facd112d
CK		 6672 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6673 16384, 0x03FFFFFF, 0, 128, 5,
6674 &fb_div, &vclk_div, &dclk_div);
6675 if (r)
6676 return r;
2539eb02
CK		 6677
6678 /* set RESET_ANTI_MUX to 0 */
6679 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6680
6681 /* set VCO_MODE to 1 */
6682 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6683
6684 /* toggle UPLL_SLEEP to 1 then back to 0 */
6685 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6686 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6687
6688 /* deassert UPLL_RESET */
6689 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6690
6691 mdelay(1);
6692
facd112d 6693 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6694 if (r)
6695 return r;
6696
6697 /* assert UPLL_RESET again */
6698 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6699
6700 /* disable spread spectrum. */
6701 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6702
6703 /* set feedback divider */
facd112d 6704 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 6705
6706 /* set ref divider to 0 */
6707 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6708
facd112d 6709 if (fb_div < 307200)
2539eb02
CK		 6710 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6711 else
6712 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6713
6714 /* set PDIV_A and PDIV_B */
6715 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 6716 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 6717 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6718
6719 /* give the PLL some time to settle */
6720 mdelay(15);
6721
6722 /* deassert PLL_RESET */
6723 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6724
6725 mdelay(15);
6726
6727 /* switch from bypass mode to normal mode */
6728 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6729
facd112d 6730 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6731 if (r)
6732 return r;
6733
6734 /* switch VCLK and DCLK selection */
6735 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6736 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6737 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6738
6739 mdelay(100);
6740
6741 return 0;
6742}
b9d305df
AD		 6743
6744static void si_pcie_gen3_enable(struct radeon_device *rdev)
6745{
6746 struct pci_dev *root = rdev->pdev->bus->self;
6747 int bridge_pos, gpu_pos;
6748 u32 speed_cntl, mask, current_data_rate;
6749 int ret, i;
6750 u16 tmp16;
6751
6752 if (radeon_pcie_gen2 == 0)
6753 return;
6754
6755 if (rdev->flags & RADEON_IS_IGP)
6756 return;
6757
6758 if (!(rdev->flags & RADEON_IS_PCIE))
6759 return;
6760
6761 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6762 if (ret != 0)
6763 return;
6764
6765 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6766 return;
6767
6768 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6769 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6770 LC_CURRENT_DATA_RATE_SHIFT;
6771 if (mask & DRM_PCIE_SPEED_80) {
6772 if (current_data_rate == 2) {
6773 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6774 return;
6775 }
6776 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6777 } else if (mask & DRM_PCIE_SPEED_50) {
6778 if (current_data_rate == 1) {
6779 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6780 return;
6781 }
6782 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6783 }
6784
6785 bridge_pos = pci_pcie_cap(root);
6786 if (!bridge_pos)
6787 return;
6788
6789 gpu_pos = pci_pcie_cap(rdev->pdev);
6790 if (!gpu_pos)
6791 return;
6792
6793 if (mask & DRM_PCIE_SPEED_80) {
6794 /* re-try equalization if gen3 is not already enabled */
6795 if (current_data_rate != 2) {
6796 u16 bridge_cfg, gpu_cfg;
6797 u16 bridge_cfg2, gpu_cfg2;
6798 u32 max_lw, current_lw, tmp;
6799
6800 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6801 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6802
6803 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6804 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6805
6806 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6807 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6808
6809 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6810 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6811 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6812
6813 if (current_lw < max_lw) {
6814 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6815 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6816 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6817 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6818 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6819 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6820 }
6821 }
6822
6823 for (i = 0; i < 10; i++) {
6824 /* check status */
6825 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6826 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6827 break;
6828
6829 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6830 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6831
6832 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6833 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6834
6835 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6836 tmp |= LC_SET_QUIESCE;
6837 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6838
6839 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6840 tmp |= LC_REDO_EQ;
6841 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6842
6843 mdelay(100);
6844
6845 /* linkctl */
6846 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6847 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6848 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6849 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6850
6851 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6852 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6853 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6854 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6855
6856 /* linkctl2 */
6857 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6858 tmp16 &= ~((1 << 4) | (7 << 9));
6859 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6860 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6861
6862 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6863 tmp16 &= ~((1 << 4) | (7 << 9));
6864 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6865 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6866
6867 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6868 tmp &= ~LC_SET_QUIESCE;
6869 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6870 }
6871 }
6872 }
6873
6874 /* set the link speed */
6875 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6876 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6877 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6878
6879 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6880 tmp16 &= ~0xf;
6881 if (mask & DRM_PCIE_SPEED_80)
6882 tmp16 |= 3; /* gen3 */
6883 else if (mask & DRM_PCIE_SPEED_50)
6884 tmp16 |= 2; /* gen2 */
6885 else
6886 tmp16 |= 1; /* gen1 */
6887 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6888
6889 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6890 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
6891 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6892
6893 for (i = 0; i < rdev->usec_timeout; i++) {
6894 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6895 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
6896 break;
6897 udelay(1);
6898 }
6899}
6900
e0bcf165
AD		 6901static void si_program_aspm(struct radeon_device *rdev)
6902{
6903 u32 data, orig;
6904 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
6905 bool disable_clkreq = false;
6906
1294d4a3
AD		 6907 if (radeon_aspm == 0)
6908 return;
6909
e0bcf165
AD		 6910 if (!(rdev->flags & RADEON_IS_PCIE))
6911 return;
6912
6913 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6914 data &= ~LC_XMIT_N_FTS_MASK;
6915 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
6916 if (orig != data)
6917 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
6918
6919 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
6920 data |= LC_GO_TO_RECOVERY;
6921 if (orig != data)
6922 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
6923
6924 orig = data = RREG32_PCIE(PCIE_P_CNTL);
6925 data |= P_IGNORE_EDB_ERR;
6926 if (orig != data)
6927 WREG32_PCIE(PCIE_P_CNTL, data);
6928
6929 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6930 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
6931 data |= LC_PMI_TO_L1_DIS;
6932 if (!disable_l0s)
6933 data |= LC_L0S_INACTIVITY(7);
6934
6935 if (!disable_l1) {
6936 data |= LC_L1_INACTIVITY(7);
6937 data &= ~LC_PMI_TO_L1_DIS;
6938 if (orig != data)
6939 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6940
6941 if (!disable_plloff_in_l1) {
6942 bool clk_req_support;
6943
6944 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6945 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6946 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6947 if (orig != data)
6948 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6949
6950 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6951 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6952 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6953 if (orig != data)
6954 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6955
6956 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6957 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6958 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6959 if (orig != data)
6960 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6961
6962 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6963 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6964 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6965 if (orig != data)
6966 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6967
6968 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
6969 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6970 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6971 if (orig != data)
6972 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6973
6974 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6975 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6976 if (orig != data)
6977 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6978
6979 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
6980 data &= ~PLL_RAMP_UP_TIME_2_MASK;
6981 if (orig != data)
6982 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
6983
6984 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
6985 data &= ~PLL_RAMP_UP_TIME_3_MASK;
6986 if (orig != data)
6987 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
6988
6989 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6990 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6991 if (orig != data)
6992 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6993
6994 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6995 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6996 if (orig != data)
6997 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6998
6999 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7000 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7001 if (orig != data)
7002 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7003
7004 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7005 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7006 if (orig != data)
7007 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7008 }
7009 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7010 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7011 data |= LC_DYN_LANES_PWR_STATE(3);
7012 if (orig != data)
7013 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7014
7015 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7016 data &= ~LS2_EXIT_TIME_MASK;
7017 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7018 data |= LS2_EXIT_TIME(5);
7019 if (orig != data)
7020 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7021
7022 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7023 data &= ~LS2_EXIT_TIME_MASK;
7024 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7025 data |= LS2_EXIT_TIME(5);
7026 if (orig != data)
7027 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7028
7029 if (!disable_clkreq) {
7030 struct pci_dev *root = rdev->pdev->bus->self;
7031 u32 lnkcap;
7032
7033 clk_req_support = false;
7034 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7035 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7036 clk_req_support = true;
7037 } else {
7038 clk_req_support = false;
7039 }
7040
7041 if (clk_req_support) {
7042 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7043 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7044 if (orig != data)
7045 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7046
7047 orig = data = RREG32(THM_CLK_CNTL);
7048 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7049 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7050 if (orig != data)
7051 WREG32(THM_CLK_CNTL, data);
7052
7053 orig = data = RREG32(MISC_CLK_CNTL);
7054 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7055 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7056 if (orig != data)
7057 WREG32(MISC_CLK_CNTL, data);
7058
7059 orig = data = RREG32(CG_CLKPIN_CNTL);
7060 data &= ~BCLK_AS_XCLK;
7061 if (orig != data)
7062 WREG32(CG_CLKPIN_CNTL, data);
7063
7064 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7065 data &= ~FORCE_BIF_REFCLK_EN;
7066 if (orig != data)
7067 WREG32(CG_CLKPIN_CNTL_2, data);
7068
7069 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7070 data &= ~MPLL_CLKOUT_SEL_MASK;
7071 data |= MPLL_CLKOUT_SEL(4);
7072 if (orig != data)
7073 WREG32(MPLL_BYPASSCLK_SEL, data);
7074
7075 orig = data = RREG32(SPLL_CNTL_MODE);
7076 data &= ~SPLL_REFCLK_SEL_MASK;
7077 if (orig != data)
7078 WREG32(SPLL_CNTL_MODE, data);
7079 }
7080 }
7081 } else {
7082 if (orig != data)
7083 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7084 }
7085
7086 orig = data = RREG32_PCIE(PCIE_CNTL2);
7087 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7088 if (orig != data)
7089 WREG32_PCIE(PCIE_CNTL2, data);
7090
7091 if (!disable_l0s) {
7092 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7093 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7094 data = RREG32_PCIE(PCIE_LC_STATUS1);
7095 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7096 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7097 data &= ~LC_L0S_INACTIVITY_MASK;
7098 if (orig != data)
7099 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7100 }
7101 }
7102 }
7103}
Linux 6.x block layer and io_uring tree(s)