projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: add clearstate init for verde power gating
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c
AD		 24#include <linux/firmware.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include <linux/module.h>
760285e7 28#include <drm/drmP.h>
43b3cd99
AD		 29#include "radeon.h"
30#include "radeon_asic.h"
760285e7 31#include <drm/radeon_drm.h>
43b3cd99
AD		 32#include "sid.h"
33#include "atom.h"
48c0c902 34#include "si_blit_shaders.h"
bd8cd539 35#include "clearstate_si.h"
43b3cd99 36
0f0de06c
AD		 37#define SI_PFP_UCODE_SIZE 2144
38#define SI_PM4_UCODE_SIZE 2144
39#define SI_CE_UCODE_SIZE 2144
40#define SI_RLC_UCODE_SIZE 2048
41#define SI_MC_UCODE_SIZE 7769
bcc7f5d2 42#define OLAND_MC_UCODE_SIZE 7863
0f0de06c
AD		 43
44MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
45MODULE_FIRMWARE("radeon/TAHITI_me.bin");
46MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
47MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
48MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
49MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
50MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
51MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
52MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
53MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
54MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
55MODULE_FIRMWARE("radeon/VERDE_me.bin");
56MODULE_FIRMWARE("radeon/VERDE_ce.bin");
57MODULE_FIRMWARE("radeon/VERDE_mc.bin");
58MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
bcc7f5d2
AD		 59MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
60MODULE_FIRMWARE("radeon/OLAND_me.bin");
61MODULE_FIRMWARE("radeon/OLAND_ce.bin");
62MODULE_FIRMWARE("radeon/OLAND_mc.bin");
63MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
c04c00b4
AD		 64MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
65MODULE_FIRMWARE("radeon/HAINAN_me.bin");
66MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
67MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
68MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
0f0de06c 69
b9d305df 70static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 71static void si_program_aspm(struct radeon_device *rdev);
25a857fb
AD		 72extern int r600_ih_ring_alloc(struct radeon_device *rdev);
73extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 74extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 75extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
76extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 77extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 78extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 79extern bool evergreen_is_display_hung(struct radeon_device *rdev);
0a96d72b 80
6d8cf000
AD		 81static const u32 verde_rlc_save_restore_register_list[] =
82{
83 (0x8000 << 16) | (0x98f4 >> 2),
84 0x00000000,
85 (0x8040 << 16) | (0x98f4 >> 2),
86 0x00000000,
87 (0x8000 << 16) | (0xe80 >> 2),
88 0x00000000,
89 (0x8040 << 16) | (0xe80 >> 2),
90 0x00000000,
91 (0x8000 << 16) | (0x89bc >> 2),
92 0x00000000,
93 (0x8040 << 16) | (0x89bc >> 2),
94 0x00000000,
95 (0x8000 << 16) | (0x8c1c >> 2),
96 0x00000000,
97 (0x8040 << 16) | (0x8c1c >> 2),
98 0x00000000,
99 (0x9c00 << 16) | (0x98f0 >> 2),
100 0x00000000,
101 (0x9c00 << 16) | (0xe7c >> 2),
102 0x00000000,
103 (0x8000 << 16) | (0x9148 >> 2),
104 0x00000000,
105 (0x8040 << 16) | (0x9148 >> 2),
106 0x00000000,
107 (0x9c00 << 16) | (0x9150 >> 2),
108 0x00000000,
109 (0x9c00 << 16) | (0x897c >> 2),
110 0x00000000,
111 (0x9c00 << 16) | (0x8d8c >> 2),
112 0x00000000,
113 (0x9c00 << 16) | (0xac54 >> 2),
114 0X00000000,
115 0x3,
116 (0x9c00 << 16) | (0x98f8 >> 2),
117 0x00000000,
118 (0x9c00 << 16) | (0x9910 >> 2),
119 0x00000000,
120 (0x9c00 << 16) | (0x9914 >> 2),
121 0x00000000,
122 (0x9c00 << 16) | (0x9918 >> 2),
123 0x00000000,
124 (0x9c00 << 16) | (0x991c >> 2),
125 0x00000000,
126 (0x9c00 << 16) | (0x9920 >> 2),
127 0x00000000,
128 (0x9c00 << 16) | (0x9924 >> 2),
129 0x00000000,
130 (0x9c00 << 16) | (0x9928 >> 2),
131 0x00000000,
132 (0x9c00 << 16) | (0x992c >> 2),
133 0x00000000,
134 (0x9c00 << 16) | (0x9930 >> 2),
135 0x00000000,
136 (0x9c00 << 16) | (0x9934 >> 2),
137 0x00000000,
138 (0x9c00 << 16) | (0x9938 >> 2),
139 0x00000000,
140 (0x9c00 << 16) | (0x993c >> 2),
141 0x00000000,
142 (0x9c00 << 16) | (0x9940 >> 2),
143 0x00000000,
144 (0x9c00 << 16) | (0x9944 >> 2),
145 0x00000000,
146 (0x9c00 << 16) | (0x9948 >> 2),
147 0x00000000,
148 (0x9c00 << 16) | (0x994c >> 2),
149 0x00000000,
150 (0x9c00 << 16) | (0x9950 >> 2),
151 0x00000000,
152 (0x9c00 << 16) | (0x9954 >> 2),
153 0x00000000,
154 (0x9c00 << 16) | (0x9958 >> 2),
155 0x00000000,
156 (0x9c00 << 16) | (0x995c >> 2),
157 0x00000000,
158 (0x9c00 << 16) | (0x9960 >> 2),
159 0x00000000,
160 (0x9c00 << 16) | (0x9964 >> 2),
161 0x00000000,
162 (0x9c00 << 16) | (0x9968 >> 2),
163 0x00000000,
164 (0x9c00 << 16) | (0x996c >> 2),
165 0x00000000,
166 (0x9c00 << 16) | (0x9970 >> 2),
167 0x00000000,
168 (0x9c00 << 16) | (0x9974 >> 2),
169 0x00000000,
170 (0x9c00 << 16) | (0x9978 >> 2),
171 0x00000000,
172 (0x9c00 << 16) | (0x997c >> 2),
173 0x00000000,
174 (0x9c00 << 16) | (0x9980 >> 2),
175 0x00000000,
176 (0x9c00 << 16) | (0x9984 >> 2),
177 0x00000000,
178 (0x9c00 << 16) | (0x9988 >> 2),
179 0x00000000,
180 (0x9c00 << 16) | (0x998c >> 2),
181 0x00000000,
182 (0x9c00 << 16) | (0x8c00 >> 2),
183 0x00000000,
184 (0x9c00 << 16) | (0x8c14 >> 2),
185 0x00000000,
186 (0x9c00 << 16) | (0x8c04 >> 2),
187 0x00000000,
188 (0x9c00 << 16) | (0x8c08 >> 2),
189 0x00000000,
190 (0x8000 << 16) | (0x9b7c >> 2),
191 0x00000000,
192 (0x8040 << 16) | (0x9b7c >> 2),
193 0x00000000,
194 (0x8000 << 16) | (0xe84 >> 2),
195 0x00000000,
196 (0x8040 << 16) | (0xe84 >> 2),
197 0x00000000,
198 (0x8000 << 16) | (0x89c0 >> 2),
199 0x00000000,
200 (0x8040 << 16) | (0x89c0 >> 2),
201 0x00000000,
202 (0x8000 << 16) | (0x914c >> 2),
203 0x00000000,
204 (0x8040 << 16) | (0x914c >> 2),
205 0x00000000,
206 (0x8000 << 16) | (0x8c20 >> 2),
207 0x00000000,
208 (0x8040 << 16) | (0x8c20 >> 2),
209 0x00000000,
210 (0x8000 << 16) | (0x9354 >> 2),
211 0x00000000,
212 (0x8040 << 16) | (0x9354 >> 2),
213 0x00000000,
214 (0x9c00 << 16) | (0x9060 >> 2),
215 0x00000000,
216 (0x9c00 << 16) | (0x9364 >> 2),
217 0x00000000,
218 (0x9c00 << 16) | (0x9100 >> 2),
219 0x00000000,
220 (0x9c00 << 16) | (0x913c >> 2),
221 0x00000000,
222 (0x8000 << 16) | (0x90e0 >> 2),
223 0x00000000,
224 (0x8000 << 16) | (0x90e4 >> 2),
225 0x00000000,
226 (0x8000 << 16) | (0x90e8 >> 2),
227 0x00000000,
228 (0x8040 << 16) | (0x90e0 >> 2),
229 0x00000000,
230 (0x8040 << 16) | (0x90e4 >> 2),
231 0x00000000,
232 (0x8040 << 16) | (0x90e8 >> 2),
233 0x00000000,
234 (0x9c00 << 16) | (0x8bcc >> 2),
235 0x00000000,
236 (0x9c00 << 16) | (0x8b24 >> 2),
237 0x00000000,
238 (0x9c00 << 16) | (0x88c4 >> 2),
239 0x00000000,
240 (0x9c00 << 16) | (0x8e50 >> 2),
241 0x00000000,
242 (0x9c00 << 16) | (0x8c0c >> 2),
243 0x00000000,
244 (0x9c00 << 16) | (0x8e58 >> 2),
245 0x00000000,
246 (0x9c00 << 16) | (0x8e5c >> 2),
247 0x00000000,
248 (0x9c00 << 16) | (0x9508 >> 2),
249 0x00000000,
250 (0x9c00 << 16) | (0x950c >> 2),
251 0x00000000,
252 (0x9c00 << 16) | (0x9494 >> 2),
253 0x00000000,
254 (0x9c00 << 16) | (0xac0c >> 2),
255 0x00000000,
256 (0x9c00 << 16) | (0xac10 >> 2),
257 0x00000000,
258 (0x9c00 << 16) | (0xac14 >> 2),
259 0x00000000,
260 (0x9c00 << 16) | (0xae00 >> 2),
261 0x00000000,
262 (0x9c00 << 16) | (0xac08 >> 2),
263 0x00000000,
264 (0x9c00 << 16) | (0x88d4 >> 2),
265 0x00000000,
266 (0x9c00 << 16) | (0x88c8 >> 2),
267 0x00000000,
268 (0x9c00 << 16) | (0x88cc >> 2),
269 0x00000000,
270 (0x9c00 << 16) | (0x89b0 >> 2),
271 0x00000000,
272 (0x9c00 << 16) | (0x8b10 >> 2),
273 0x00000000,
274 (0x9c00 << 16) | (0x8a14 >> 2),
275 0x00000000,
276 (0x9c00 << 16) | (0x9830 >> 2),
277 0x00000000,
278 (0x9c00 << 16) | (0x9834 >> 2),
279 0x00000000,
280 (0x9c00 << 16) | (0x9838 >> 2),
281 0x00000000,
282 (0x9c00 << 16) | (0x9a10 >> 2),
283 0x00000000,
284 (0x8000 << 16) | (0x9870 >> 2),
285 0x00000000,
286 (0x8000 << 16) | (0x9874 >> 2),
287 0x00000000,
288 (0x8001 << 16) | (0x9870 >> 2),
289 0x00000000,
290 (0x8001 << 16) | (0x9874 >> 2),
291 0x00000000,
292 (0x8040 << 16) | (0x9870 >> 2),
293 0x00000000,
294 (0x8040 << 16) | (0x9874 >> 2),
295 0x00000000,
296 (0x8041 << 16) | (0x9870 >> 2),
297 0x00000000,
298 (0x8041 << 16) | (0x9874 >> 2),
299 0x00000000,
300 0x00000000
301};
302
205996c0
AD		 303static const u32 tahiti_golden_rlc_registers[] =
304{
305 0xc424, 0xffffffff, 0x00601005,
306 0xc47c, 0xffffffff, 0x10104040,
307 0xc488, 0xffffffff, 0x0100000a,
308 0xc314, 0xffffffff, 0x00000800,
309 0xc30c, 0xffffffff, 0x800000f4,
310 0xf4a8, 0xffffffff, 0x00000000
311};
312
313static const u32 tahiti_golden_registers[] =
314{
315 0x9a10, 0x00010000, 0x00018208,
316 0x9830, 0xffffffff, 0x00000000,
317 0x9834, 0xf00fffff, 0x00000400,
318 0x9838, 0x0002021c, 0x00020200,
319 0xc78, 0x00000080, 0x00000000,
320 0xd030, 0x000300c0, 0x00800040,
321 0xd830, 0x000300c0, 0x00800040,
322 0x5bb0, 0x000000f0, 0x00000070,
323 0x5bc0, 0x00200000, 0x50100000,
324 0x7030, 0x31000311, 0x00000011,
325 0x277c, 0x00000003, 0x000007ff,
326 0x240c, 0x000007ff, 0x00000000,
327 0x8a14, 0xf000001f, 0x00000007,
328 0x8b24, 0xffffffff, 0x00ffffff,
329 0x8b10, 0x0000ff0f, 0x00000000,
330 0x28a4c, 0x07ffffff, 0x4e000000,
331 0x28350, 0x3f3f3fff, 0x2a00126a,
332 0x30, 0x000000ff, 0x0040,
333 0x34, 0x00000040, 0x00004040,
334 0x9100, 0x07ffffff, 0x03000000,
335 0x8e88, 0x01ff1f3f, 0x00000000,
336 0x8e84, 0x01ff1f3f, 0x00000000,
337 0x9060, 0x0000007f, 0x00000020,
338 0x9508, 0x00010000, 0x00010000,
339 0xac14, 0x00000200, 0x000002fb,
340 0xac10, 0xffffffff, 0x0000543b,
341 0xac0c, 0xffffffff, 0xa9210876,
342 0x88d0, 0xffffffff, 0x000fff40,
343 0x88d4, 0x0000001f, 0x00000010,
344 0x1410, 0x20000000, 0x20fffed8,
345 0x15c0, 0x000c0fc0, 0x000c0400
346};
347
348static const u32 tahiti_golden_registers2[] =
349{
350 0xc64, 0x00000001, 0x00000001
351};
352
353static const u32 pitcairn_golden_rlc_registers[] =
354{
355 0xc424, 0xffffffff, 0x00601004,
356 0xc47c, 0xffffffff, 0x10102020,
357 0xc488, 0xffffffff, 0x01000020,
358 0xc314, 0xffffffff, 0x00000800,
359 0xc30c, 0xffffffff, 0x800000a4
360};
361
362static const u32 pitcairn_golden_registers[] =
363{
364 0x9a10, 0x00010000, 0x00018208,
365 0x9830, 0xffffffff, 0x00000000,
366 0x9834, 0xf00fffff, 0x00000400,
367 0x9838, 0x0002021c, 0x00020200,
368 0xc78, 0x00000080, 0x00000000,
369 0xd030, 0x000300c0, 0x00800040,
370 0xd830, 0x000300c0, 0x00800040,
371 0x5bb0, 0x000000f0, 0x00000070,
372 0x5bc0, 0x00200000, 0x50100000,
373 0x7030, 0x31000311, 0x00000011,
374 0x2ae4, 0x00073ffe, 0x000022a2,
375 0x240c, 0x000007ff, 0x00000000,
376 0x8a14, 0xf000001f, 0x00000007,
377 0x8b24, 0xffffffff, 0x00ffffff,
378 0x8b10, 0x0000ff0f, 0x00000000,
379 0x28a4c, 0x07ffffff, 0x4e000000,
380 0x28350, 0x3f3f3fff, 0x2a00126a,
381 0x30, 0x000000ff, 0x0040,
382 0x34, 0x00000040, 0x00004040,
383 0x9100, 0x07ffffff, 0x03000000,
384 0x9060, 0x0000007f, 0x00000020,
385 0x9508, 0x00010000, 0x00010000,
386 0xac14, 0x000003ff, 0x000000f7,
387 0xac10, 0xffffffff, 0x00000000,
388 0xac0c, 0xffffffff, 0x32761054,
389 0x88d4, 0x0000001f, 0x00000010,
390 0x15c0, 0x000c0fc0, 0x000c0400
391};
392
393static const u32 verde_golden_rlc_registers[] =
394{
395 0xc424, 0xffffffff, 0x033f1005,
396 0xc47c, 0xffffffff, 0x10808020,
397 0xc488, 0xffffffff, 0x00800008,
398 0xc314, 0xffffffff, 0x00001000,
399 0xc30c, 0xffffffff, 0x80010014
400};
401
402static const u32 verde_golden_registers[] =
403{
404 0x9a10, 0x00010000, 0x00018208,
405 0x9830, 0xffffffff, 0x00000000,
406 0x9834, 0xf00fffff, 0x00000400,
407 0x9838, 0x0002021c, 0x00020200,
408 0xc78, 0x00000080, 0x00000000,
409 0xd030, 0x000300c0, 0x00800040,
410 0xd030, 0x000300c0, 0x00800040,
411 0xd830, 0x000300c0, 0x00800040,
412 0xd830, 0x000300c0, 0x00800040,
413 0x5bb0, 0x000000f0, 0x00000070,
414 0x5bc0, 0x00200000, 0x50100000,
415 0x7030, 0x31000311, 0x00000011,
416 0x2ae4, 0x00073ffe, 0x000022a2,
417 0x2ae4, 0x00073ffe, 0x000022a2,
418 0x2ae4, 0x00073ffe, 0x000022a2,
419 0x240c, 0x000007ff, 0x00000000,
420 0x240c, 0x000007ff, 0x00000000,
421 0x240c, 0x000007ff, 0x00000000,
422 0x8a14, 0xf000001f, 0x00000007,
423 0x8a14, 0xf000001f, 0x00000007,
424 0x8a14, 0xf000001f, 0x00000007,
425 0x8b24, 0xffffffff, 0x00ffffff,
426 0x8b10, 0x0000ff0f, 0x00000000,
427 0x28a4c, 0x07ffffff, 0x4e000000,
428 0x28350, 0x3f3f3fff, 0x0000124a,
429 0x28350, 0x3f3f3fff, 0x0000124a,
430 0x28350, 0x3f3f3fff, 0x0000124a,
431 0x30, 0x000000ff, 0x0040,
432 0x34, 0x00000040, 0x00004040,
433 0x9100, 0x07ffffff, 0x03000000,
434 0x9100, 0x07ffffff, 0x03000000,
435 0x8e88, 0x01ff1f3f, 0x00000000,
436 0x8e88, 0x01ff1f3f, 0x00000000,
437 0x8e88, 0x01ff1f3f, 0x00000000,
438 0x8e84, 0x01ff1f3f, 0x00000000,
439 0x8e84, 0x01ff1f3f, 0x00000000,
440 0x8e84, 0x01ff1f3f, 0x00000000,
441 0x9060, 0x0000007f, 0x00000020,
442 0x9508, 0x00010000, 0x00010000,
443 0xac14, 0x000003ff, 0x00000003,
444 0xac14, 0x000003ff, 0x00000003,
445 0xac14, 0x000003ff, 0x00000003,
446 0xac10, 0xffffffff, 0x00000000,
447 0xac10, 0xffffffff, 0x00000000,
448 0xac10, 0xffffffff, 0x00000000,
449 0xac0c, 0xffffffff, 0x00001032,
450 0xac0c, 0xffffffff, 0x00001032,
451 0xac0c, 0xffffffff, 0x00001032,
452 0x88d4, 0x0000001f, 0x00000010,
453 0x88d4, 0x0000001f, 0x00000010,
454 0x88d4, 0x0000001f, 0x00000010,
455 0x15c0, 0x000c0fc0, 0x000c0400
456};
457
458static const u32 oland_golden_rlc_registers[] =
459{
460 0xc424, 0xffffffff, 0x00601005,
461 0xc47c, 0xffffffff, 0x10104040,
462 0xc488, 0xffffffff, 0x0100000a,
463 0xc314, 0xffffffff, 0x00000800,
464 0xc30c, 0xffffffff, 0x800000f4
465};
466
467static const u32 oland_golden_registers[] =
468{
469 0x9a10, 0x00010000, 0x00018208,
470 0x9830, 0xffffffff, 0x00000000,
471 0x9834, 0xf00fffff, 0x00000400,
472 0x9838, 0x0002021c, 0x00020200,
473 0xc78, 0x00000080, 0x00000000,
474 0xd030, 0x000300c0, 0x00800040,
475 0xd830, 0x000300c0, 0x00800040,
476 0x5bb0, 0x000000f0, 0x00000070,
477 0x5bc0, 0x00200000, 0x50100000,
478 0x7030, 0x31000311, 0x00000011,
479 0x2ae4, 0x00073ffe, 0x000022a2,
480 0x240c, 0x000007ff, 0x00000000,
481 0x8a14, 0xf000001f, 0x00000007,
482 0x8b24, 0xffffffff, 0x00ffffff,
483 0x8b10, 0x0000ff0f, 0x00000000,
484 0x28a4c, 0x07ffffff, 0x4e000000,
485 0x28350, 0x3f3f3fff, 0x00000082,
486 0x30, 0x000000ff, 0x0040,
487 0x34, 0x00000040, 0x00004040,
488 0x9100, 0x07ffffff, 0x03000000,
489 0x9060, 0x0000007f, 0x00000020,
490 0x9508, 0x00010000, 0x00010000,
491 0xac14, 0x000003ff, 0x000000f3,
492 0xac10, 0xffffffff, 0x00000000,
493 0xac0c, 0xffffffff, 0x00003210,
494 0x88d4, 0x0000001f, 0x00000010,
495 0x15c0, 0x000c0fc0, 0x000c0400
496};
497
fffbdda4
AD		 498static const u32 hainan_golden_registers[] =
499{
500 0x9a10, 0x00010000, 0x00018208,
501 0x9830, 0xffffffff, 0x00000000,
502 0x9834, 0xf00fffff, 0x00000400,
503 0x9838, 0x0002021c, 0x00020200,
504 0xd0c0, 0xff000fff, 0x00000100,
505 0xd030, 0x000300c0, 0x00800040,
506 0xd8c0, 0xff000fff, 0x00000100,
507 0xd830, 0x000300c0, 0x00800040,
508 0x2ae4, 0x00073ffe, 0x000022a2,
509 0x240c, 0x000007ff, 0x00000000,
510 0x8a14, 0xf000001f, 0x00000007,
511 0x8b24, 0xffffffff, 0x00ffffff,
512 0x8b10, 0x0000ff0f, 0x00000000,
513 0x28a4c, 0x07ffffff, 0x4e000000,
514 0x28350, 0x3f3f3fff, 0x00000000,
515 0x30, 0x000000ff, 0x0040,
516 0x34, 0x00000040, 0x00004040,
517 0x9100, 0x03e00000, 0x03600000,
518 0x9060, 0x0000007f, 0x00000020,
519 0x9508, 0x00010000, 0x00010000,
520 0xac14, 0x000003ff, 0x000000f1,
521 0xac10, 0xffffffff, 0x00000000,
522 0xac0c, 0xffffffff, 0x00003210,
523 0x88d4, 0x0000001f, 0x00000010,
524 0x15c0, 0x000c0fc0, 0x000c0400
525};
526
527static const u32 hainan_golden_registers2[] =
528{
529 0x98f8, 0xffffffff, 0x02010001
530};
531
205996c0
AD		 532static const u32 tahiti_mgcg_cgcg_init[] =
533{
534 0xc400, 0xffffffff, 0xfffffffc,
535 0x802c, 0xffffffff, 0xe0000000,
536 0x9a60, 0xffffffff, 0x00000100,
537 0x92a4, 0xffffffff, 0x00000100,
538 0xc164, 0xffffffff, 0x00000100,
539 0x9774, 0xffffffff, 0x00000100,
540 0x8984, 0xffffffff, 0x06000100,
541 0x8a18, 0xffffffff, 0x00000100,
542 0x92a0, 0xffffffff, 0x00000100,
543 0xc380, 0xffffffff, 0x00000100,
544 0x8b28, 0xffffffff, 0x00000100,
545 0x9144, 0xffffffff, 0x00000100,
546 0x8d88, 0xffffffff, 0x00000100,
547 0x8d8c, 0xffffffff, 0x00000100,
548 0x9030, 0xffffffff, 0x00000100,
549 0x9034, 0xffffffff, 0x00000100,
550 0x9038, 0xffffffff, 0x00000100,
551 0x903c, 0xffffffff, 0x00000100,
552 0xad80, 0xffffffff, 0x00000100,
553 0xac54, 0xffffffff, 0x00000100,
554 0x897c, 0xffffffff, 0x06000100,
555 0x9868, 0xffffffff, 0x00000100,
556 0x9510, 0xffffffff, 0x00000100,
557 0xaf04, 0xffffffff, 0x00000100,
558 0xae04, 0xffffffff, 0x00000100,
559 0x949c, 0xffffffff, 0x00000100,
560 0x802c, 0xffffffff, 0xe0000000,
561 0x9160, 0xffffffff, 0x00010000,
562 0x9164, 0xffffffff, 0x00030002,
563 0x9168, 0xffffffff, 0x00040007,
564 0x916c, 0xffffffff, 0x00060005,
565 0x9170, 0xffffffff, 0x00090008,
566 0x9174, 0xffffffff, 0x00020001,
567 0x9178, 0xffffffff, 0x00040003,
568 0x917c, 0xffffffff, 0x00000007,
569 0x9180, 0xffffffff, 0x00060005,
570 0x9184, 0xffffffff, 0x00090008,
571 0x9188, 0xffffffff, 0x00030002,
572 0x918c, 0xffffffff, 0x00050004,
573 0x9190, 0xffffffff, 0x00000008,
574 0x9194, 0xffffffff, 0x00070006,
575 0x9198, 0xffffffff, 0x000a0009,
576 0x919c, 0xffffffff, 0x00040003,
577 0x91a0, 0xffffffff, 0x00060005,
578 0x91a4, 0xffffffff, 0x00000009,
579 0x91a8, 0xffffffff, 0x00080007,
580 0x91ac, 0xffffffff, 0x000b000a,
581 0x91b0, 0xffffffff, 0x00050004,
582 0x91b4, 0xffffffff, 0x00070006,
583 0x91b8, 0xffffffff, 0x0008000b,
584 0x91bc, 0xffffffff, 0x000a0009,
585 0x91c0, 0xffffffff, 0x000d000c,
586 0x91c4, 0xffffffff, 0x00060005,
587 0x91c8, 0xffffffff, 0x00080007,
588 0x91cc, 0xffffffff, 0x0000000b,
589 0x91d0, 0xffffffff, 0x000a0009,
590 0x91d4, 0xffffffff, 0x000d000c,
591 0x91d8, 0xffffffff, 0x00070006,
592 0x91dc, 0xffffffff, 0x00090008,
593 0x91e0, 0xffffffff, 0x0000000c,
594 0x91e4, 0xffffffff, 0x000b000a,
595 0x91e8, 0xffffffff, 0x000e000d,
596 0x91ec, 0xffffffff, 0x00080007,
597 0x91f0, 0xffffffff, 0x000a0009,
598 0x91f4, 0xffffffff, 0x0000000d,
599 0x91f8, 0xffffffff, 0x000c000b,
600 0x91fc, 0xffffffff, 0x000f000e,
601 0x9200, 0xffffffff, 0x00090008,
602 0x9204, 0xffffffff, 0x000b000a,
603 0x9208, 0xffffffff, 0x000c000f,
604 0x920c, 0xffffffff, 0x000e000d,
605 0x9210, 0xffffffff, 0x00110010,
606 0x9214, 0xffffffff, 0x000a0009,
607 0x9218, 0xffffffff, 0x000c000b,
608 0x921c, 0xffffffff, 0x0000000f,
609 0x9220, 0xffffffff, 0x000e000d,
610 0x9224, 0xffffffff, 0x00110010,
611 0x9228, 0xffffffff, 0x000b000a,
612 0x922c, 0xffffffff, 0x000d000c,
613 0x9230, 0xffffffff, 0x00000010,
614 0x9234, 0xffffffff, 0x000f000e,
615 0x9238, 0xffffffff, 0x00120011,
616 0x923c, 0xffffffff, 0x000c000b,
617 0x9240, 0xffffffff, 0x000e000d,
618 0x9244, 0xffffffff, 0x00000011,
619 0x9248, 0xffffffff, 0x0010000f,
620 0x924c, 0xffffffff, 0x00130012,
621 0x9250, 0xffffffff, 0x000d000c,
622 0x9254, 0xffffffff, 0x000f000e,
623 0x9258, 0xffffffff, 0x00100013,
624 0x925c, 0xffffffff, 0x00120011,
625 0x9260, 0xffffffff, 0x00150014,
626 0x9264, 0xffffffff, 0x000e000d,
627 0x9268, 0xffffffff, 0x0010000f,
628 0x926c, 0xffffffff, 0x00000013,
629 0x9270, 0xffffffff, 0x00120011,
630 0x9274, 0xffffffff, 0x00150014,
631 0x9278, 0xffffffff, 0x000f000e,
632 0x927c, 0xffffffff, 0x00110010,
633 0x9280, 0xffffffff, 0x00000014,
634 0x9284, 0xffffffff, 0x00130012,
635 0x9288, 0xffffffff, 0x00160015,
636 0x928c, 0xffffffff, 0x0010000f,
637 0x9290, 0xffffffff, 0x00120011,
638 0x9294, 0xffffffff, 0x00000015,
639 0x9298, 0xffffffff, 0x00140013,
640 0x929c, 0xffffffff, 0x00170016,
641 0x9150, 0xffffffff, 0x96940200,
642 0x8708, 0xffffffff, 0x00900100,
643 0xc478, 0xffffffff, 0x00000080,
644 0xc404, 0xffffffff, 0x0020003f,
645 0x30, 0xffffffff, 0x0000001c,
646 0x34, 0x000f0000, 0x000f0000,
647 0x160c, 0xffffffff, 0x00000100,
648 0x1024, 0xffffffff, 0x00000100,
649 0x102c, 0x00000101, 0x00000000,
650 0x20a8, 0xffffffff, 0x00000104,
651 0x264c, 0x000c0000, 0x000c0000,
652 0x2648, 0x000c0000, 0x000c0000,
653 0x55e4, 0xff000fff, 0x00000100,
654 0x55e8, 0x00000001, 0x00000001,
655 0x2f50, 0x00000001, 0x00000001,
656 0x30cc, 0xc0000fff, 0x00000104,
657 0xc1e4, 0x00000001, 0x00000001,
658 0xd0c0, 0xfffffff0, 0x00000100,
659 0xd8c0, 0xfffffff0, 0x00000100
660};
661
662static const u32 pitcairn_mgcg_cgcg_init[] =
663{
664 0xc400, 0xffffffff, 0xfffffffc,
665 0x802c, 0xffffffff, 0xe0000000,
666 0x9a60, 0xffffffff, 0x00000100,
667 0x92a4, 0xffffffff, 0x00000100,
668 0xc164, 0xffffffff, 0x00000100,
669 0x9774, 0xffffffff, 0x00000100,
670 0x8984, 0xffffffff, 0x06000100,
671 0x8a18, 0xffffffff, 0x00000100,
672 0x92a0, 0xffffffff, 0x00000100,
673 0xc380, 0xffffffff, 0x00000100,
674 0x8b28, 0xffffffff, 0x00000100,
675 0x9144, 0xffffffff, 0x00000100,
676 0x8d88, 0xffffffff, 0x00000100,
677 0x8d8c, 0xffffffff, 0x00000100,
678 0x9030, 0xffffffff, 0x00000100,
679 0x9034, 0xffffffff, 0x00000100,
680 0x9038, 0xffffffff, 0x00000100,
681 0x903c, 0xffffffff, 0x00000100,
682 0xad80, 0xffffffff, 0x00000100,
683 0xac54, 0xffffffff, 0x00000100,
684 0x897c, 0xffffffff, 0x06000100,
685 0x9868, 0xffffffff, 0x00000100,
686 0x9510, 0xffffffff, 0x00000100,
687 0xaf04, 0xffffffff, 0x00000100,
688 0xae04, 0xffffffff, 0x00000100,
689 0x949c, 0xffffffff, 0x00000100,
690 0x802c, 0xffffffff, 0xe0000000,
691 0x9160, 0xffffffff, 0x00010000,
692 0x9164, 0xffffffff, 0x00030002,
693 0x9168, 0xffffffff, 0x00040007,
694 0x916c, 0xffffffff, 0x00060005,
695 0x9170, 0xffffffff, 0x00090008,
696 0x9174, 0xffffffff, 0x00020001,
697 0x9178, 0xffffffff, 0x00040003,
698 0x917c, 0xffffffff, 0x00000007,
699 0x9180, 0xffffffff, 0x00060005,
700 0x9184, 0xffffffff, 0x00090008,
701 0x9188, 0xffffffff, 0x00030002,
702 0x918c, 0xffffffff, 0x00050004,
703 0x9190, 0xffffffff, 0x00000008,
704 0x9194, 0xffffffff, 0x00070006,
705 0x9198, 0xffffffff, 0x000a0009,
706 0x919c, 0xffffffff, 0x00040003,
707 0x91a0, 0xffffffff, 0x00060005,
708 0x91a4, 0xffffffff, 0x00000009,
709 0x91a8, 0xffffffff, 0x00080007,
710 0x91ac, 0xffffffff, 0x000b000a,
711 0x91b0, 0xffffffff, 0x00050004,
712 0x91b4, 0xffffffff, 0x00070006,
713 0x91b8, 0xffffffff, 0x0008000b,
714 0x91bc, 0xffffffff, 0x000a0009,
715 0x91c0, 0xffffffff, 0x000d000c,
716 0x9200, 0xffffffff, 0x00090008,
717 0x9204, 0xffffffff, 0x000b000a,
718 0x9208, 0xffffffff, 0x000c000f,
719 0x920c, 0xffffffff, 0x000e000d,
720 0x9210, 0xffffffff, 0x00110010,
721 0x9214, 0xffffffff, 0x000a0009,
722 0x9218, 0xffffffff, 0x000c000b,
723 0x921c, 0xffffffff, 0x0000000f,
724 0x9220, 0xffffffff, 0x000e000d,
725 0x9224, 0xffffffff, 0x00110010,
726 0x9228, 0xffffffff, 0x000b000a,
727 0x922c, 0xffffffff, 0x000d000c,
728 0x9230, 0xffffffff, 0x00000010,
729 0x9234, 0xffffffff, 0x000f000e,
730 0x9238, 0xffffffff, 0x00120011,
731 0x923c, 0xffffffff, 0x000c000b,
732 0x9240, 0xffffffff, 0x000e000d,
733 0x9244, 0xffffffff, 0x00000011,
734 0x9248, 0xffffffff, 0x0010000f,
735 0x924c, 0xffffffff, 0x00130012,
736 0x9250, 0xffffffff, 0x000d000c,
737 0x9254, 0xffffffff, 0x000f000e,
738 0x9258, 0xffffffff, 0x00100013,
739 0x925c, 0xffffffff, 0x00120011,
740 0x9260, 0xffffffff, 0x00150014,
741 0x9150, 0xffffffff, 0x96940200,
742 0x8708, 0xffffffff, 0x00900100,
743 0xc478, 0xffffffff, 0x00000080,
744 0xc404, 0xffffffff, 0x0020003f,
745 0x30, 0xffffffff, 0x0000001c,
746 0x34, 0x000f0000, 0x000f0000,
747 0x160c, 0xffffffff, 0x00000100,
748 0x1024, 0xffffffff, 0x00000100,
749 0x102c, 0x00000101, 0x00000000,
750 0x20a8, 0xffffffff, 0x00000104,
751 0x55e4, 0xff000fff, 0x00000100,
752 0x55e8, 0x00000001, 0x00000001,
753 0x2f50, 0x00000001, 0x00000001,
754 0x30cc, 0xc0000fff, 0x00000104,
755 0xc1e4, 0x00000001, 0x00000001,
756 0xd0c0, 0xfffffff0, 0x00000100,
757 0xd8c0, 0xfffffff0, 0x00000100
758};
759
760static const u32 verde_mgcg_cgcg_init[] =
761{
762 0xc400, 0xffffffff, 0xfffffffc,
763 0x802c, 0xffffffff, 0xe0000000,
764 0x9a60, 0xffffffff, 0x00000100,
765 0x92a4, 0xffffffff, 0x00000100,
766 0xc164, 0xffffffff, 0x00000100,
767 0x9774, 0xffffffff, 0x00000100,
768 0x8984, 0xffffffff, 0x06000100,
769 0x8a18, 0xffffffff, 0x00000100,
770 0x92a0, 0xffffffff, 0x00000100,
771 0xc380, 0xffffffff, 0x00000100,
772 0x8b28, 0xffffffff, 0x00000100,
773 0x9144, 0xffffffff, 0x00000100,
774 0x8d88, 0xffffffff, 0x00000100,
775 0x8d8c, 0xffffffff, 0x00000100,
776 0x9030, 0xffffffff, 0x00000100,
777 0x9034, 0xffffffff, 0x00000100,
778 0x9038, 0xffffffff, 0x00000100,
779 0x903c, 0xffffffff, 0x00000100,
780 0xad80, 0xffffffff, 0x00000100,
781 0xac54, 0xffffffff, 0x00000100,
782 0x897c, 0xffffffff, 0x06000100,
783 0x9868, 0xffffffff, 0x00000100,
784 0x9510, 0xffffffff, 0x00000100,
785 0xaf04, 0xffffffff, 0x00000100,
786 0xae04, 0xffffffff, 0x00000100,
787 0x949c, 0xffffffff, 0x00000100,
788 0x802c, 0xffffffff, 0xe0000000,
789 0x9160, 0xffffffff, 0x00010000,
790 0x9164, 0xffffffff, 0x00030002,
791 0x9168, 0xffffffff, 0x00040007,
792 0x916c, 0xffffffff, 0x00060005,
793 0x9170, 0xffffffff, 0x00090008,
794 0x9174, 0xffffffff, 0x00020001,
795 0x9178, 0xffffffff, 0x00040003,
796 0x917c, 0xffffffff, 0x00000007,
797 0x9180, 0xffffffff, 0x00060005,
798 0x9184, 0xffffffff, 0x00090008,
799 0x9188, 0xffffffff, 0x00030002,
800 0x918c, 0xffffffff, 0x00050004,
801 0x9190, 0xffffffff, 0x00000008,
802 0x9194, 0xffffffff, 0x00070006,
803 0x9198, 0xffffffff, 0x000a0009,
804 0x919c, 0xffffffff, 0x00040003,
805 0x91a0, 0xffffffff, 0x00060005,
806 0x91a4, 0xffffffff, 0x00000009,
807 0x91a8, 0xffffffff, 0x00080007,
808 0x91ac, 0xffffffff, 0x000b000a,
809 0x91b0, 0xffffffff, 0x00050004,
810 0x91b4, 0xffffffff, 0x00070006,
811 0x91b8, 0xffffffff, 0x0008000b,
812 0x91bc, 0xffffffff, 0x000a0009,
813 0x91c0, 0xffffffff, 0x000d000c,
814 0x9200, 0xffffffff, 0x00090008,
815 0x9204, 0xffffffff, 0x000b000a,
816 0x9208, 0xffffffff, 0x000c000f,
817 0x920c, 0xffffffff, 0x000e000d,
818 0x9210, 0xffffffff, 0x00110010,
819 0x9214, 0xffffffff, 0x000a0009,
820 0x9218, 0xffffffff, 0x000c000b,
821 0x921c, 0xffffffff, 0x0000000f,
822 0x9220, 0xffffffff, 0x000e000d,
823 0x9224, 0xffffffff, 0x00110010,
824 0x9228, 0xffffffff, 0x000b000a,
825 0x922c, 0xffffffff, 0x000d000c,
826 0x9230, 0xffffffff, 0x00000010,
827 0x9234, 0xffffffff, 0x000f000e,
828 0x9238, 0xffffffff, 0x00120011,
829 0x923c, 0xffffffff, 0x000c000b,
830 0x9240, 0xffffffff, 0x000e000d,
831 0x9244, 0xffffffff, 0x00000011,
832 0x9248, 0xffffffff, 0x0010000f,
833 0x924c, 0xffffffff, 0x00130012,
834 0x9250, 0xffffffff, 0x000d000c,
835 0x9254, 0xffffffff, 0x000f000e,
836 0x9258, 0xffffffff, 0x00100013,
837 0x925c, 0xffffffff, 0x00120011,
838 0x9260, 0xffffffff, 0x00150014,
839 0x9150, 0xffffffff, 0x96940200,
840 0x8708, 0xffffffff, 0x00900100,
841 0xc478, 0xffffffff, 0x00000080,
842 0xc404, 0xffffffff, 0x0020003f,
843 0x30, 0xffffffff, 0x0000001c,
844 0x34, 0x000f0000, 0x000f0000,
845 0x160c, 0xffffffff, 0x00000100,
846 0x1024, 0xffffffff, 0x00000100,
847 0x102c, 0x00000101, 0x00000000,
848 0x20a8, 0xffffffff, 0x00000104,
849 0x264c, 0x000c0000, 0x000c0000,
850 0x2648, 0x000c0000, 0x000c0000,
851 0x55e4, 0xff000fff, 0x00000100,
852 0x55e8, 0x00000001, 0x00000001,
853 0x2f50, 0x00000001, 0x00000001,
854 0x30cc, 0xc0000fff, 0x00000104,
855 0xc1e4, 0x00000001, 0x00000001,
856 0xd0c0, 0xfffffff0, 0x00000100,
857 0xd8c0, 0xfffffff0, 0x00000100
858};
859
860static const u32 oland_mgcg_cgcg_init[] =
861{
862 0xc400, 0xffffffff, 0xfffffffc,
863 0x802c, 0xffffffff, 0xe0000000,
864 0x9a60, 0xffffffff, 0x00000100,
865 0x92a4, 0xffffffff, 0x00000100,
866 0xc164, 0xffffffff, 0x00000100,
867 0x9774, 0xffffffff, 0x00000100,
868 0x8984, 0xffffffff, 0x06000100,
869 0x8a18, 0xffffffff, 0x00000100,
870 0x92a0, 0xffffffff, 0x00000100,
871 0xc380, 0xffffffff, 0x00000100,
872 0x8b28, 0xffffffff, 0x00000100,
873 0x9144, 0xffffffff, 0x00000100,
874 0x8d88, 0xffffffff, 0x00000100,
875 0x8d8c, 0xffffffff, 0x00000100,
876 0x9030, 0xffffffff, 0x00000100,
877 0x9034, 0xffffffff, 0x00000100,
878 0x9038, 0xffffffff, 0x00000100,
879 0x903c, 0xffffffff, 0x00000100,
880 0xad80, 0xffffffff, 0x00000100,
881 0xac54, 0xffffffff, 0x00000100,
882 0x897c, 0xffffffff, 0x06000100,
883 0x9868, 0xffffffff, 0x00000100,
884 0x9510, 0xffffffff, 0x00000100,
885 0xaf04, 0xffffffff, 0x00000100,
886 0xae04, 0xffffffff, 0x00000100,
887 0x949c, 0xffffffff, 0x00000100,
888 0x802c, 0xffffffff, 0xe0000000,
889 0x9160, 0xffffffff, 0x00010000,
890 0x9164, 0xffffffff, 0x00030002,
891 0x9168, 0xffffffff, 0x00040007,
892 0x916c, 0xffffffff, 0x00060005,
893 0x9170, 0xffffffff, 0x00090008,
894 0x9174, 0xffffffff, 0x00020001,
895 0x9178, 0xffffffff, 0x00040003,
896 0x917c, 0xffffffff, 0x00000007,
897 0x9180, 0xffffffff, 0x00060005,
898 0x9184, 0xffffffff, 0x00090008,
899 0x9188, 0xffffffff, 0x00030002,
900 0x918c, 0xffffffff, 0x00050004,
901 0x9190, 0xffffffff, 0x00000008,
902 0x9194, 0xffffffff, 0x00070006,
903 0x9198, 0xffffffff, 0x000a0009,
904 0x919c, 0xffffffff, 0x00040003,
905 0x91a0, 0xffffffff, 0x00060005,
906 0x91a4, 0xffffffff, 0x00000009,
907 0x91a8, 0xffffffff, 0x00080007,
908 0x91ac, 0xffffffff, 0x000b000a,
909 0x91b0, 0xffffffff, 0x00050004,
910 0x91b4, 0xffffffff, 0x00070006,
911 0x91b8, 0xffffffff, 0x0008000b,
912 0x91bc, 0xffffffff, 0x000a0009,
913 0x91c0, 0xffffffff, 0x000d000c,
914 0x91c4, 0xffffffff, 0x00060005,
915 0x91c8, 0xffffffff, 0x00080007,
916 0x91cc, 0xffffffff, 0x0000000b,
917 0x91d0, 0xffffffff, 0x000a0009,
918 0x91d4, 0xffffffff, 0x000d000c,
919 0x9150, 0xffffffff, 0x96940200,
920 0x8708, 0xffffffff, 0x00900100,
921 0xc478, 0xffffffff, 0x00000080,
922 0xc404, 0xffffffff, 0x0020003f,
923 0x30, 0xffffffff, 0x0000001c,
924 0x34, 0x000f0000, 0x000f0000,
925 0x160c, 0xffffffff, 0x00000100,
926 0x1024, 0xffffffff, 0x00000100,
927 0x102c, 0x00000101, 0x00000000,
928 0x20a8, 0xffffffff, 0x00000104,
929 0x264c, 0x000c0000, 0x000c0000,
930 0x2648, 0x000c0000, 0x000c0000,
931 0x55e4, 0xff000fff, 0x00000100,
932 0x55e8, 0x00000001, 0x00000001,
933 0x2f50, 0x00000001, 0x00000001,
934 0x30cc, 0xc0000fff, 0x00000104,
935 0xc1e4, 0x00000001, 0x00000001,
936 0xd0c0, 0xfffffff0, 0x00000100,
937 0xd8c0, 0xfffffff0, 0x00000100
938};
939
fffbdda4
AD		 940static const u32 hainan_mgcg_cgcg_init[] =
941{
942 0xc400, 0xffffffff, 0xfffffffc,
943 0x802c, 0xffffffff, 0xe0000000,
944 0x9a60, 0xffffffff, 0x00000100,
945 0x92a4, 0xffffffff, 0x00000100,
946 0xc164, 0xffffffff, 0x00000100,
947 0x9774, 0xffffffff, 0x00000100,
948 0x8984, 0xffffffff, 0x06000100,
949 0x8a18, 0xffffffff, 0x00000100,
950 0x92a0, 0xffffffff, 0x00000100,
951 0xc380, 0xffffffff, 0x00000100,
952 0x8b28, 0xffffffff, 0x00000100,
953 0x9144, 0xffffffff, 0x00000100,
954 0x8d88, 0xffffffff, 0x00000100,
955 0x8d8c, 0xffffffff, 0x00000100,
956 0x9030, 0xffffffff, 0x00000100,
957 0x9034, 0xffffffff, 0x00000100,
958 0x9038, 0xffffffff, 0x00000100,
959 0x903c, 0xffffffff, 0x00000100,
960 0xad80, 0xffffffff, 0x00000100,
961 0xac54, 0xffffffff, 0x00000100,
962 0x897c, 0xffffffff, 0x06000100,
963 0x9868, 0xffffffff, 0x00000100,
964 0x9510, 0xffffffff, 0x00000100,
965 0xaf04, 0xffffffff, 0x00000100,
966 0xae04, 0xffffffff, 0x00000100,
967 0x949c, 0xffffffff, 0x00000100,
968 0x802c, 0xffffffff, 0xe0000000,
969 0x9160, 0xffffffff, 0x00010000,
970 0x9164, 0xffffffff, 0x00030002,
971 0x9168, 0xffffffff, 0x00040007,
972 0x916c, 0xffffffff, 0x00060005,
973 0x9170, 0xffffffff, 0x00090008,
974 0x9174, 0xffffffff, 0x00020001,
975 0x9178, 0xffffffff, 0x00040003,
976 0x917c, 0xffffffff, 0x00000007,
977 0x9180, 0xffffffff, 0x00060005,
978 0x9184, 0xffffffff, 0x00090008,
979 0x9188, 0xffffffff, 0x00030002,
980 0x918c, 0xffffffff, 0x00050004,
981 0x9190, 0xffffffff, 0x00000008,
982 0x9194, 0xffffffff, 0x00070006,
983 0x9198, 0xffffffff, 0x000a0009,
984 0x919c, 0xffffffff, 0x00040003,
985 0x91a0, 0xffffffff, 0x00060005,
986 0x91a4, 0xffffffff, 0x00000009,
987 0x91a8, 0xffffffff, 0x00080007,
988 0x91ac, 0xffffffff, 0x000b000a,
989 0x91b0, 0xffffffff, 0x00050004,
990 0x91b4, 0xffffffff, 0x00070006,
991 0x91b8, 0xffffffff, 0x0008000b,
992 0x91bc, 0xffffffff, 0x000a0009,
993 0x91c0, 0xffffffff, 0x000d000c,
994 0x91c4, 0xffffffff, 0x00060005,
995 0x91c8, 0xffffffff, 0x00080007,
996 0x91cc, 0xffffffff, 0x0000000b,
997 0x91d0, 0xffffffff, 0x000a0009,
998 0x91d4, 0xffffffff, 0x000d000c,
999 0x9150, 0xffffffff, 0x96940200,
1000 0x8708, 0xffffffff, 0x00900100,
1001 0xc478, 0xffffffff, 0x00000080,
1002 0xc404, 0xffffffff, 0x0020003f,
1003 0x30, 0xffffffff, 0x0000001c,
1004 0x34, 0x000f0000, 0x000f0000,
1005 0x160c, 0xffffffff, 0x00000100,
1006 0x1024, 0xffffffff, 0x00000100,
1007 0x20a8, 0xffffffff, 0x00000104,
1008 0x264c, 0x000c0000, 0x000c0000,
1009 0x2648, 0x000c0000, 0x000c0000,
1010 0x2f50, 0x00000001, 0x00000001,
1011 0x30cc, 0xc0000fff, 0x00000104,
1012 0xc1e4, 0x00000001, 0x00000001,
1013 0xd0c0, 0xfffffff0, 0x00000100,
1014 0xd8c0, 0xfffffff0, 0x00000100
1015};
1016
205996c0
AD		 1017static u32 verde_pg_init[] =
1018{
1019 0x353c, 0xffffffff, 0x40000,
1020 0x3538, 0xffffffff, 0x200010ff,
1021 0x353c, 0xffffffff, 0x0,
1022 0x353c, 0xffffffff, 0x0,
1023 0x353c, 0xffffffff, 0x0,
1024 0x353c, 0xffffffff, 0x0,
1025 0x353c, 0xffffffff, 0x0,
1026 0x353c, 0xffffffff, 0x7007,
1027 0x3538, 0xffffffff, 0x300010ff,
1028 0x353c, 0xffffffff, 0x0,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x400000,
1034 0x3538, 0xffffffff, 0x100010ff,
1035 0x353c, 0xffffffff, 0x0,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x120200,
1041 0x3538, 0xffffffff, 0x500010ff,
1042 0x353c, 0xffffffff, 0x0,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x1e1e16,
1048 0x3538, 0xffffffff, 0x600010ff,
1049 0x353c, 0xffffffff, 0x0,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x171f1e,
1055 0x3538, 0xffffffff, 0x700010ff,
1056 0x353c, 0xffffffff, 0x0,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x3538, 0xffffffff, 0x9ff,
1063 0x3500, 0xffffffff, 0x0,
1064 0x3504, 0xffffffff, 0x10000800,
1065 0x3504, 0xffffffff, 0xf,
1066 0x3504, 0xffffffff, 0xf,
1067 0x3500, 0xffffffff, 0x4,
1068 0x3504, 0xffffffff, 0x1000051e,
1069 0x3504, 0xffffffff, 0xffff,
1070 0x3504, 0xffffffff, 0xffff,
1071 0x3500, 0xffffffff, 0x8,
1072 0x3504, 0xffffffff, 0x80500,
1073 0x3500, 0xffffffff, 0x12,
1074 0x3504, 0xffffffff, 0x9050c,
1075 0x3500, 0xffffffff, 0x1d,
1076 0x3504, 0xffffffff, 0xb052c,
1077 0x3500, 0xffffffff, 0x2a,
1078 0x3504, 0xffffffff, 0x1053e,
1079 0x3500, 0xffffffff, 0x2d,
1080 0x3504, 0xffffffff, 0x10546,
1081 0x3500, 0xffffffff, 0x30,
1082 0x3504, 0xffffffff, 0xa054e,
1083 0x3500, 0xffffffff, 0x3c,
1084 0x3504, 0xffffffff, 0x1055f,
1085 0x3500, 0xffffffff, 0x3f,
1086 0x3504, 0xffffffff, 0x10567,
1087 0x3500, 0xffffffff, 0x42,
1088 0x3504, 0xffffffff, 0x1056f,
1089 0x3500, 0xffffffff, 0x45,
1090 0x3504, 0xffffffff, 0x10572,
1091 0x3500, 0xffffffff, 0x48,
1092 0x3504, 0xffffffff, 0x20575,
1093 0x3500, 0xffffffff, 0x4c,
1094 0x3504, 0xffffffff, 0x190801,
1095 0x3500, 0xffffffff, 0x67,
1096 0x3504, 0xffffffff, 0x1082a,
1097 0x3500, 0xffffffff, 0x6a,
1098 0x3504, 0xffffffff, 0x1b082d,
1099 0x3500, 0xffffffff, 0x87,
1100 0x3504, 0xffffffff, 0x310851,
1101 0x3500, 0xffffffff, 0xba,
1102 0x3504, 0xffffffff, 0x891,
1103 0x3500, 0xffffffff, 0xbc,
1104 0x3504, 0xffffffff, 0x893,
1105 0x3500, 0xffffffff, 0xbe,
1106 0x3504, 0xffffffff, 0x20895,
1107 0x3500, 0xffffffff, 0xc2,
1108 0x3504, 0xffffffff, 0x20899,
1109 0x3500, 0xffffffff, 0xc6,
1110 0x3504, 0xffffffff, 0x2089d,
1111 0x3500, 0xffffffff, 0xca,
1112 0x3504, 0xffffffff, 0x8a1,
1113 0x3500, 0xffffffff, 0xcc,
1114 0x3504, 0xffffffff, 0x8a3,
1115 0x3500, 0xffffffff, 0xce,
1116 0x3504, 0xffffffff, 0x308a5,
1117 0x3500, 0xffffffff, 0xd3,
1118 0x3504, 0xffffffff, 0x6d08cd,
1119 0x3500, 0xffffffff, 0x142,
1120 0x3504, 0xffffffff, 0x2000095a,
1121 0x3504, 0xffffffff, 0x1,
1122 0x3500, 0xffffffff, 0x144,
1123 0x3504, 0xffffffff, 0x301f095b,
1124 0x3500, 0xffffffff, 0x165,
1125 0x3504, 0xffffffff, 0xc094d,
1126 0x3500, 0xffffffff, 0x173,
1127 0x3504, 0xffffffff, 0xf096d,
1128 0x3500, 0xffffffff, 0x184,
1129 0x3504, 0xffffffff, 0x15097f,
1130 0x3500, 0xffffffff, 0x19b,
1131 0x3504, 0xffffffff, 0xc0998,
1132 0x3500, 0xffffffff, 0x1a9,
1133 0x3504, 0xffffffff, 0x409a7,
1134 0x3500, 0xffffffff, 0x1af,
1135 0x3504, 0xffffffff, 0xcdc,
1136 0x3500, 0xffffffff, 0x1b1,
1137 0x3504, 0xffffffff, 0x800,
1138 0x3508, 0xffffffff, 0x6c9b2000,
1139 0x3510, 0xfc00, 0x2000,
1140 0x3544, 0xffffffff, 0xfc0,
1141 0x28d4, 0x00000100, 0x100
1142};
1143
1144static void si_init_golden_registers(struct radeon_device *rdev)
1145{
1146 switch (rdev->family) {
1147 case CHIP_TAHITI:
1148 radeon_program_register_sequence(rdev,
1149 tahiti_golden_registers,
1150 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1151 radeon_program_register_sequence(rdev,
1152 tahiti_golden_rlc_registers,
1153 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1154 radeon_program_register_sequence(rdev,
1155 tahiti_mgcg_cgcg_init,
1156 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1157 radeon_program_register_sequence(rdev,
1158 tahiti_golden_registers2,
1159 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1160 break;
1161 case CHIP_PITCAIRN:
1162 radeon_program_register_sequence(rdev,
1163 pitcairn_golden_registers,
1164 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1165 radeon_program_register_sequence(rdev,
1166 pitcairn_golden_rlc_registers,
1167 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1168 radeon_program_register_sequence(rdev,
1169 pitcairn_mgcg_cgcg_init,
1170 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1171 break;
1172 case CHIP_VERDE:
1173 radeon_program_register_sequence(rdev,
1174 verde_golden_registers,
1175 (const u32)ARRAY_SIZE(verde_golden_registers));
1176 radeon_program_register_sequence(rdev,
1177 verde_golden_rlc_registers,
1178 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1179 radeon_program_register_sequence(rdev,
1180 verde_mgcg_cgcg_init,
1181 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1182 radeon_program_register_sequence(rdev,
1183 verde_pg_init,
1184 (const u32)ARRAY_SIZE(verde_pg_init));
1185 break;
1186 case CHIP_OLAND:
1187 radeon_program_register_sequence(rdev,
1188 oland_golden_registers,
1189 (const u32)ARRAY_SIZE(oland_golden_registers));
1190 radeon_program_register_sequence(rdev,
1191 oland_golden_rlc_registers,
1192 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1193 radeon_program_register_sequence(rdev,
1194 oland_mgcg_cgcg_init,
1195 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1196 break;
fffbdda4
AD		 1197 case CHIP_HAINAN:
1198 radeon_program_register_sequence(rdev,
1199 hainan_golden_registers,
1200 (const u32)ARRAY_SIZE(hainan_golden_registers));
1201 radeon_program_register_sequence(rdev,
1202 hainan_golden_registers2,
1203 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1204 radeon_program_register_sequence(rdev,
1205 hainan_mgcg_cgcg_init,
1206 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1207 break;
205996c0
AD		 1208 default:
1209 break;
1210 }
1211}
1212
454d2e2a
AD		 1213#define PCIE_BUS_CLK 10000
1214#define TCLK (PCIE_BUS_CLK / 10)
1215
1216/**
1217 * si_get_xclk - get the xclk
1218 *
1219 * @rdev: radeon_device pointer
1220 *
1221 * Returns the reference clock used by the gfx engine
1222 * (SI).
1223 */
1224u32 si_get_xclk(struct radeon_device *rdev)
1225{
1226 u32 reference_clock = rdev->clock.spll.reference_freq;
1227 u32 tmp;
1228
1229 tmp = RREG32(CG_CLKPIN_CNTL_2);
1230 if (tmp & MUX_TCLK_TO_XCLK)
1231 return TCLK;
1232
1233 tmp = RREG32(CG_CLKPIN_CNTL);
1234 if (tmp & XTALIN_DIVIDE)
1235 return reference_clock / 4;
1236
1237 return reference_clock;
1238}
1239
1bd47d2e
AD		 1240/* get temperature in millidegrees */
1241int si_get_temp(struct radeon_device *rdev)
1242{
1243 u32 temp;
1244 int actual_temp = 0;
1245
1246 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1247 CTF_TEMP_SHIFT;
1248
1249 if (temp & 0x200)
1250 actual_temp = 255;
1251 else
1252 actual_temp = temp & 0x1ff;
1253
1254 actual_temp = (actual_temp * 1000);
1255
1256 return actual_temp;
1257}
1258
8b074dd6
AD		 1259#define TAHITI_IO_MC_REGS_SIZE 36
1260
1261static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1262 {0x0000006f, 0x03044000},
1263 {0x00000070, 0x0480c018},
1264 {0x00000071, 0x00000040},
1265 {0x00000072, 0x01000000},
1266 {0x00000074, 0x000000ff},
1267 {0x00000075, 0x00143400},
1268 {0x00000076, 0x08ec0800},
1269 {0x00000077, 0x040000cc},
1270 {0x00000079, 0x00000000},
1271 {0x0000007a, 0x21000409},
1272 {0x0000007c, 0x00000000},
1273 {0x0000007d, 0xe8000000},
1274 {0x0000007e, 0x044408a8},
1275 {0x0000007f, 0x00000003},
1276 {0x00000080, 0x00000000},
1277 {0x00000081, 0x01000000},
1278 {0x00000082, 0x02000000},
1279 {0x00000083, 0x00000000},
1280 {0x00000084, 0xe3f3e4f4},
1281 {0x00000085, 0x00052024},
1282 {0x00000087, 0x00000000},
1283 {0x00000088, 0x66036603},
1284 {0x00000089, 0x01000000},
1285 {0x0000008b, 0x1c0a0000},
1286 {0x0000008c, 0xff010000},
1287 {0x0000008e, 0xffffefff},
1288 {0x0000008f, 0xfff3efff},
1289 {0x00000090, 0xfff3efbf},
1290 {0x00000094, 0x00101101},
1291 {0x00000095, 0x00000fff},
1292 {0x00000096, 0x00116fff},
1293 {0x00000097, 0x60010000},
1294 {0x00000098, 0x10010000},
1295 {0x00000099, 0x00006000},
1296 {0x0000009a, 0x00001000},
1297 {0x0000009f, 0x00a77400}
1298};
1299
1300static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1301 {0x0000006f, 0x03044000},
1302 {0x00000070, 0x0480c018},
1303 {0x00000071, 0x00000040},
1304 {0x00000072, 0x01000000},
1305 {0x00000074, 0x000000ff},
1306 {0x00000075, 0x00143400},
1307 {0x00000076, 0x08ec0800},
1308 {0x00000077, 0x040000cc},
1309 {0x00000079, 0x00000000},
1310 {0x0000007a, 0x21000409},
1311 {0x0000007c, 0x00000000},
1312 {0x0000007d, 0xe8000000},
1313 {0x0000007e, 0x044408a8},
1314 {0x0000007f, 0x00000003},
1315 {0x00000080, 0x00000000},
1316 {0x00000081, 0x01000000},
1317 {0x00000082, 0x02000000},
1318 {0x00000083, 0x00000000},
1319 {0x00000084, 0xe3f3e4f4},
1320 {0x00000085, 0x00052024},
1321 {0x00000087, 0x00000000},
1322 {0x00000088, 0x66036603},
1323 {0x00000089, 0x01000000},
1324 {0x0000008b, 0x1c0a0000},
1325 {0x0000008c, 0xff010000},
1326 {0x0000008e, 0xffffefff},
1327 {0x0000008f, 0xfff3efff},
1328 {0x00000090, 0xfff3efbf},
1329 {0x00000094, 0x00101101},
1330 {0x00000095, 0x00000fff},
1331 {0x00000096, 0x00116fff},
1332 {0x00000097, 0x60010000},
1333 {0x00000098, 0x10010000},
1334 {0x00000099, 0x00006000},
1335 {0x0000009a, 0x00001000},
1336 {0x0000009f, 0x00a47400}
1337};
1338
1339static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1340 {0x0000006f, 0x03044000},
1341 {0x00000070, 0x0480c018},
1342 {0x00000071, 0x00000040},
1343 {0x00000072, 0x01000000},
1344 {0x00000074, 0x000000ff},
1345 {0x00000075, 0x00143400},
1346 {0x00000076, 0x08ec0800},
1347 {0x00000077, 0x040000cc},
1348 {0x00000079, 0x00000000},
1349 {0x0000007a, 0x21000409},
1350 {0x0000007c, 0x00000000},
1351 {0x0000007d, 0xe8000000},
1352 {0x0000007e, 0x044408a8},
1353 {0x0000007f, 0x00000003},
1354 {0x00000080, 0x00000000},
1355 {0x00000081, 0x01000000},
1356 {0x00000082, 0x02000000},
1357 {0x00000083, 0x00000000},
1358 {0x00000084, 0xe3f3e4f4},
1359 {0x00000085, 0x00052024},
1360 {0x00000087, 0x00000000},
1361 {0x00000088, 0x66036603},
1362 {0x00000089, 0x01000000},
1363 {0x0000008b, 0x1c0a0000},
1364 {0x0000008c, 0xff010000},
1365 {0x0000008e, 0xffffefff},
1366 {0x0000008f, 0xfff3efff},
1367 {0x00000090, 0xfff3efbf},
1368 {0x00000094, 0x00101101},
1369 {0x00000095, 0x00000fff},
1370 {0x00000096, 0x00116fff},
1371 {0x00000097, 0x60010000},
1372 {0x00000098, 0x10010000},
1373 {0x00000099, 0x00006000},
1374 {0x0000009a, 0x00001000},
1375 {0x0000009f, 0x00a37400}
1376};
1377
bcc7f5d2
AD		 1378static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1379 {0x0000006f, 0x03044000},
1380 {0x00000070, 0x0480c018},
1381 {0x00000071, 0x00000040},
1382 {0x00000072, 0x01000000},
1383 {0x00000074, 0x000000ff},
1384 {0x00000075, 0x00143400},
1385 {0x00000076, 0x08ec0800},
1386 {0x00000077, 0x040000cc},
1387 {0x00000079, 0x00000000},
1388 {0x0000007a, 0x21000409},
1389 {0x0000007c, 0x00000000},
1390 {0x0000007d, 0xe8000000},
1391 {0x0000007e, 0x044408a8},
1392 {0x0000007f, 0x00000003},
1393 {0x00000080, 0x00000000},
1394 {0x00000081, 0x01000000},
1395 {0x00000082, 0x02000000},
1396 {0x00000083, 0x00000000},
1397 {0x00000084, 0xe3f3e4f4},
1398 {0x00000085, 0x00052024},
1399 {0x00000087, 0x00000000},
1400 {0x00000088, 0x66036603},
1401 {0x00000089, 0x01000000},
1402 {0x0000008b, 0x1c0a0000},
1403 {0x0000008c, 0xff010000},
1404 {0x0000008e, 0xffffefff},
1405 {0x0000008f, 0xfff3efff},
1406 {0x00000090, 0xfff3efbf},
1407 {0x00000094, 0x00101101},
1408 {0x00000095, 0x00000fff},
1409 {0x00000096, 0x00116fff},
1410 {0x00000097, 0x60010000},
1411 {0x00000098, 0x10010000},
1412 {0x00000099, 0x00006000},
1413 {0x0000009a, 0x00001000},
1414 {0x0000009f, 0x00a17730}
1415};
1416
c04c00b4
AD		 1417static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1418 {0x0000006f, 0x03044000},
1419 {0x00000070, 0x0480c018},
1420 {0x00000071, 0x00000040},
1421 {0x00000072, 0x01000000},
1422 {0x00000074, 0x000000ff},
1423 {0x00000075, 0x00143400},
1424 {0x00000076, 0x08ec0800},
1425 {0x00000077, 0x040000cc},
1426 {0x00000079, 0x00000000},
1427 {0x0000007a, 0x21000409},
1428 {0x0000007c, 0x00000000},
1429 {0x0000007d, 0xe8000000},
1430 {0x0000007e, 0x044408a8},
1431 {0x0000007f, 0x00000003},
1432 {0x00000080, 0x00000000},
1433 {0x00000081, 0x01000000},
1434 {0x00000082, 0x02000000},
1435 {0x00000083, 0x00000000},
1436 {0x00000084, 0xe3f3e4f4},
1437 {0x00000085, 0x00052024},
1438 {0x00000087, 0x00000000},
1439 {0x00000088, 0x66036603},
1440 {0x00000089, 0x01000000},
1441 {0x0000008b, 0x1c0a0000},
1442 {0x0000008c, 0xff010000},
1443 {0x0000008e, 0xffffefff},
1444 {0x0000008f, 0xfff3efff},
1445 {0x00000090, 0xfff3efbf},
1446 {0x00000094, 0x00101101},
1447 {0x00000095, 0x00000fff},
1448 {0x00000096, 0x00116fff},
1449 {0x00000097, 0x60010000},
1450 {0x00000098, 0x10010000},
1451 {0x00000099, 0x00006000},
1452 {0x0000009a, 0x00001000},
1453 {0x0000009f, 0x00a07730}
1454};
1455
8b074dd6
AD		 1456/* ucode loading */
1457static int si_mc_load_microcode(struct radeon_device *rdev)
1458{
1459 const __be32 *fw_data;
1460 u32 running, blackout = 0;
1461 u32 *io_mc_regs;
1462 int i, ucode_size, regs_size;
1463
1464 if (!rdev->mc_fw)
1465 return -EINVAL;
1466
1467 switch (rdev->family) {
1468 case CHIP_TAHITI:
1469 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1470 ucode_size = SI_MC_UCODE_SIZE;
1471 regs_size = TAHITI_IO_MC_REGS_SIZE;
1472 break;
1473 case CHIP_PITCAIRN:
1474 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1475 ucode_size = SI_MC_UCODE_SIZE;
1476 regs_size = TAHITI_IO_MC_REGS_SIZE;
1477 break;
1478 case CHIP_VERDE:
1479 default:
1480 io_mc_regs = (u32 *)&verde_io_mc_regs;
1481 ucode_size = SI_MC_UCODE_SIZE;
1482 regs_size = TAHITI_IO_MC_REGS_SIZE;
1483 break;
bcc7f5d2
AD		 1484 case CHIP_OLAND:
1485 io_mc_regs = (u32 *)&oland_io_mc_regs;
1486 ucode_size = OLAND_MC_UCODE_SIZE;
1487 regs_size = TAHITI_IO_MC_REGS_SIZE;
1488 break;
c04c00b4
AD		 1489 case CHIP_HAINAN:
1490 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1491 ucode_size = OLAND_MC_UCODE_SIZE;
1492 regs_size = TAHITI_IO_MC_REGS_SIZE;
1493 break;
8b074dd6
AD		 1494 }
1495
1496 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1497
1498 if (running == 0) {
1499 if (running) {
1500 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1501 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1502 }
1503
1504 /* reset the engine and set to writable */
1505 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1506 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1507
1508 /* load mc io regs */
1509 for (i = 0; i < regs_size; i++) {
1510 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1511 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1512 }
1513 /* load the MC ucode */
1514 fw_data = (const __be32 *)rdev->mc_fw->data;
1515 for (i = 0; i < ucode_size; i++)
1516 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1517
1518 /* put the engine back into the active state */
1519 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1520 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1521 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1522
1523 /* wait for training to complete */
1524 for (i = 0; i < rdev->usec_timeout; i++) {
1525 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1526 break;
1527 udelay(1);
1528 }
1529 for (i = 0; i < rdev->usec_timeout; i++) {
1530 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1531 break;
1532 udelay(1);
1533 }
1534
1535 if (running)
1536 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1537 }
1538
1539 return 0;
1540}
1541
0f0de06c
AD		 1542static int si_init_microcode(struct radeon_device *rdev)
1543{
1544 struct platform_device *pdev;
1545 const char *chip_name;
1546 const char *rlc_chip_name;
1547 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1548 char fw_name[30];
1549 int err;
1550
1551 DRM_DEBUG("\n");
1552
1553 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1554 err = IS_ERR(pdev);
1555 if (err) {
1556 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1557 return -EINVAL;
1558 }
1559
1560 switch (rdev->family) {
1561 case CHIP_TAHITI:
1562 chip_name = "TAHITI";
1563 rlc_chip_name = "TAHITI";
1564 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1565 me_req_size = SI_PM4_UCODE_SIZE * 4;
1566 ce_req_size = SI_CE_UCODE_SIZE * 4;
1567 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1568 mc_req_size = SI_MC_UCODE_SIZE * 4;
1569 break;
1570 case CHIP_PITCAIRN:
1571 chip_name = "PITCAIRN";
1572 rlc_chip_name = "PITCAIRN";
1573 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1574 me_req_size = SI_PM4_UCODE_SIZE * 4;
1575 ce_req_size = SI_CE_UCODE_SIZE * 4;
1576 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1577 mc_req_size = SI_MC_UCODE_SIZE * 4;
1578 break;
1579 case CHIP_VERDE:
1580 chip_name = "VERDE";
1581 rlc_chip_name = "VERDE";
1582 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1583 me_req_size = SI_PM4_UCODE_SIZE * 4;
1584 ce_req_size = SI_CE_UCODE_SIZE * 4;
1585 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1586 mc_req_size = SI_MC_UCODE_SIZE * 4;
1587 break;
bcc7f5d2
AD		 1588 case CHIP_OLAND:
1589 chip_name = "OLAND";
1590 rlc_chip_name = "OLAND";
1591 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1592 me_req_size = SI_PM4_UCODE_SIZE * 4;
1593 ce_req_size = SI_CE_UCODE_SIZE * 4;
1594 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1595 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1596 break;
c04c00b4
AD		 1597 case CHIP_HAINAN:
1598 chip_name = "HAINAN";
1599 rlc_chip_name = "HAINAN";
1600 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1601 me_req_size = SI_PM4_UCODE_SIZE * 4;
1602 ce_req_size = SI_CE_UCODE_SIZE * 4;
1603 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1604 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1605 break;
0f0de06c
AD		 1606 default: BUG();
1607 }
1608
1609 DRM_INFO("Loading %s Microcode\n", chip_name);
1610
1611 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1612 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1613 if (err)
1614 goto out;
1615 if (rdev->pfp_fw->size != pfp_req_size) {
1616 printk(KERN_ERR
1617 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1618 rdev->pfp_fw->size, fw_name);
1619 err = -EINVAL;
1620 goto out;
1621 }
1622
1623 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1624 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1625 if (err)
1626 goto out;
1627 if (rdev->me_fw->size != me_req_size) {
1628 printk(KERN_ERR
1629 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1630 rdev->me_fw->size, fw_name);
1631 err = -EINVAL;
1632 }
1633
1634 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1635 err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
1636 if (err)
1637 goto out;
1638 if (rdev->ce_fw->size != ce_req_size) {
1639 printk(KERN_ERR
1640 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1641 rdev->ce_fw->size, fw_name);
1642 err = -EINVAL;
1643 }
1644
1645 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1646 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1647 if (err)
1648 goto out;
1649 if (rdev->rlc_fw->size != rlc_req_size) {
1650 printk(KERN_ERR
1651 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1652 rdev->rlc_fw->size, fw_name);
1653 err = -EINVAL;
1654 }
1655
1656 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1657 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
1658 if (err)
1659 goto out;
1660 if (rdev->mc_fw->size != mc_req_size) {
1661 printk(KERN_ERR
1662 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1663 rdev->mc_fw->size, fw_name);
1664 err = -EINVAL;
1665 }
1666
1667out:
1668 platform_device_unregister(pdev);
1669
1670 if (err) {
1671 if (err != -EINVAL)
1672 printk(KERN_ERR
1673 "si_cp: Failed to load firmware \"%s\"\n",
1674 fw_name);
1675 release_firmware(rdev->pfp_fw);
1676 rdev->pfp_fw = NULL;
1677 release_firmware(rdev->me_fw);
1678 rdev->me_fw = NULL;
1679 release_firmware(rdev->ce_fw);
1680 rdev->ce_fw = NULL;
1681 release_firmware(rdev->rlc_fw);
1682 rdev->rlc_fw = NULL;
1683 release_firmware(rdev->mc_fw);
1684 rdev->mc_fw = NULL;
1685 }
1686 return err;
1687}
1688
43b3cd99
AD		 1689/* watermark setup */
1690static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1691 struct radeon_crtc *radeon_crtc,
1692 struct drm_display_mode *mode,
1693 struct drm_display_mode *other_mode)
1694{
1695 u32 tmp;
1696 /*
1697 * Line Buffer Setup
1698 * There are 3 line buffers, each one shared by 2 display controllers.
1699 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1700 * the display controllers. The paritioning is done via one of four
1701 * preset allocations specified in bits 21:20:
1702 * 0 - half lb
1703 * 2 - whole lb, other crtc must be disabled
1704 */
1705 /* this can get tricky if we have two large displays on a paired group
1706 * of crtcs. Ideally for multiple large displays we'd assign them to
1707 * non-linked crtcs for maximum line buffer allocation.
1708 */
1709 if (radeon_crtc->base.enabled && mode) {
1710 if (other_mode)
1711 tmp = 0; /* 1/2 */
1712 else
1713 tmp = 2; /* whole */
1714 } else
1715 tmp = 0;
1716
1717 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1718 DC_LB_MEMORY_CONFIG(tmp));
1719
1720 if (radeon_crtc->base.enabled && mode) {
1721 switch (tmp) {
1722 case 0:
1723 default:
1724 return 4096 * 2;
1725 case 2:
1726 return 8192 * 2;
1727 }
1728 }
1729
1730 /* controller not enabled, so no lb used */
1731 return 0;
1732}
1733
ca7db22b 1734static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1735{
1736 u32 tmp = RREG32(MC_SHARED_CHMAP);
1737
1738 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1739 case 0:
1740 default:
1741 return 1;
1742 case 1:
1743 return 2;
1744 case 2:
1745 return 4;
1746 case 3:
1747 return 8;
1748 case 4:
1749 return 3;
1750 case 5:
1751 return 6;
1752 case 6:
1753 return 10;
1754 case 7:
1755 return 12;
1756 case 8:
1757 return 16;
1758 }
1759}
1760
1761struct dce6_wm_params {
1762 u32 dram_channels; /* number of dram channels */
1763 u32 yclk; /* bandwidth per dram data pin in kHz */
1764 u32 sclk; /* engine clock in kHz */
1765 u32 disp_clk; /* display clock in kHz */
1766 u32 src_width; /* viewport width */
1767 u32 active_time; /* active display time in ns */
1768 u32 blank_time; /* blank time in ns */
1769 bool interlaced; /* mode is interlaced */
1770 fixed20_12 vsc; /* vertical scale ratio */
1771 u32 num_heads; /* number of active crtcs */
1772 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1773 u32 lb_size; /* line buffer allocated to pipe */
1774 u32 vtaps; /* vertical scaler taps */
1775};
1776
1777static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1778{
1779 /* Calculate raw DRAM Bandwidth */
1780 fixed20_12 dram_efficiency; /* 0.7 */
1781 fixed20_12 yclk, dram_channels, bandwidth;
1782 fixed20_12 a;
1783
1784 a.full = dfixed_const(1000);
1785 yclk.full = dfixed_const(wm->yclk);
1786 yclk.full = dfixed_div(yclk, a);
1787 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1788 a.full = dfixed_const(10);
1789 dram_efficiency.full = dfixed_const(7);
1790 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1791 bandwidth.full = dfixed_mul(dram_channels, yclk);
1792 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1793
1794 return dfixed_trunc(bandwidth);
1795}
1796
1797static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1798{
1799 /* Calculate DRAM Bandwidth and the part allocated to display. */
1800 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1801 fixed20_12 yclk, dram_channels, bandwidth;
1802 fixed20_12 a;
1803
1804 a.full = dfixed_const(1000);
1805 yclk.full = dfixed_const(wm->yclk);
1806 yclk.full = dfixed_div(yclk, a);
1807 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1808 a.full = dfixed_const(10);
1809 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1810 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1811 bandwidth.full = dfixed_mul(dram_channels, yclk);
1812 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1813
1814 return dfixed_trunc(bandwidth);
1815}
1816
1817static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1818{
1819 /* Calculate the display Data return Bandwidth */
1820 fixed20_12 return_efficiency; /* 0.8 */
1821 fixed20_12 sclk, bandwidth;
1822 fixed20_12 a;
1823
1824 a.full = dfixed_const(1000);
1825 sclk.full = dfixed_const(wm->sclk);
1826 sclk.full = dfixed_div(sclk, a);
1827 a.full = dfixed_const(10);
1828 return_efficiency.full = dfixed_const(8);
1829 return_efficiency.full = dfixed_div(return_efficiency, a);
1830 a.full = dfixed_const(32);
1831 bandwidth.full = dfixed_mul(a, sclk);
1832 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1833
1834 return dfixed_trunc(bandwidth);
1835}
1836
1837static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1838{
1839 return 32;
1840}
1841
1842static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1843{
1844 /* Calculate the DMIF Request Bandwidth */
1845 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1846 fixed20_12 disp_clk, sclk, bandwidth;
1847 fixed20_12 a, b1, b2;
1848 u32 min_bandwidth;
1849
1850 a.full = dfixed_const(1000);
1851 disp_clk.full = dfixed_const(wm->disp_clk);
1852 disp_clk.full = dfixed_div(disp_clk, a);
1853 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1854 b1.full = dfixed_mul(a, disp_clk);
1855
1856 a.full = dfixed_const(1000);
1857 sclk.full = dfixed_const(wm->sclk);
1858 sclk.full = dfixed_div(sclk, a);
1859 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1860 b2.full = dfixed_mul(a, sclk);
1861
1862 a.full = dfixed_const(10);
1863 disp_clk_request_efficiency.full = dfixed_const(8);
1864 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1865
1866 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1867
1868 a.full = dfixed_const(min_bandwidth);
1869 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1870
1871 return dfixed_trunc(bandwidth);
1872}
1873
1874static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1875{
1876 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1877 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1878 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1879 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1880
1881 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1882}
1883
1884static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1885{
1886 /* Calculate the display mode Average Bandwidth
1887 * DisplayMode should contain the source and destination dimensions,
1888 * timing, etc.
1889 */
1890 fixed20_12 bpp;
1891 fixed20_12 line_time;
1892 fixed20_12 src_width;
1893 fixed20_12 bandwidth;
1894 fixed20_12 a;
1895
1896 a.full = dfixed_const(1000);
1897 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1898 line_time.full = dfixed_div(line_time, a);
1899 bpp.full = dfixed_const(wm->bytes_per_pixel);
1900 src_width.full = dfixed_const(wm->src_width);
1901 bandwidth.full = dfixed_mul(src_width, bpp);
1902 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1903 bandwidth.full = dfixed_div(bandwidth, line_time);
1904
1905 return dfixed_trunc(bandwidth);
1906}
1907
1908static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1909{
1910 /* First calcualte the latency in ns */
1911 u32 mc_latency = 2000; /* 2000 ns. */
1912 u32 available_bandwidth = dce6_available_bandwidth(wm);
1913 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1914 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1915 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1916 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1917 (wm->num_heads * cursor_line_pair_return_time);
1918 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1919 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1920 u32 tmp, dmif_size = 12288;
1921 fixed20_12 a, b, c;
1922
1923 if (wm->num_heads == 0)
1924 return 0;
1925
1926 a.full = dfixed_const(2);
1927 b.full = dfixed_const(1);
1928 if ((wm->vsc.full > a.full) ||
1929 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1930 (wm->vtaps >= 5) ||
1931 ((wm->vsc.full >= a.full) && wm->interlaced))
1932 max_src_lines_per_dst_line = 4;
1933 else
1934 max_src_lines_per_dst_line = 2;
1935
1936 a.full = dfixed_const(available_bandwidth);
1937 b.full = dfixed_const(wm->num_heads);
1938 a.full = dfixed_div(a, b);
1939
1940 b.full = dfixed_const(mc_latency + 512);
1941 c.full = dfixed_const(wm->disp_clk);
1942 b.full = dfixed_div(b, c);
1943
1944 c.full = dfixed_const(dmif_size);
1945 b.full = dfixed_div(c, b);
1946
1947 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1948
1949 b.full = dfixed_const(1000);
1950 c.full = dfixed_const(wm->disp_clk);
1951 b.full = dfixed_div(c, b);
1952 c.full = dfixed_const(wm->bytes_per_pixel);
1953 b.full = dfixed_mul(b, c);
1954
1955 lb_fill_bw = min(tmp, dfixed_trunc(b));
1956
1957 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1958 b.full = dfixed_const(1000);
1959 c.full = dfixed_const(lb_fill_bw);
1960 b.full = dfixed_div(c, b);
1961 a.full = dfixed_div(a, b);
1962 line_fill_time = dfixed_trunc(a);
1963
1964 if (line_fill_time < wm->active_time)
1965 return latency;
1966 else
1967 return latency + (line_fill_time - wm->active_time);
1968
1969}
1970
1971static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1972{
1973 if (dce6_average_bandwidth(wm) <=
1974 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1975 return true;
1976 else
1977 return false;
1978};
1979
1980static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
1981{
1982 if (dce6_average_bandwidth(wm) <=
1983 (dce6_available_bandwidth(wm) / wm->num_heads))
1984 return true;
1985 else
1986 return false;
1987};
1988
1989static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
1990{
1991 u32 lb_partitions = wm->lb_size / wm->src_width;
1992 u32 line_time = wm->active_time + wm->blank_time;
1993 u32 latency_tolerant_lines;
1994 u32 latency_hiding;
1995 fixed20_12 a;
1996
1997 a.full = dfixed_const(1);
1998 if (wm->vsc.full > a.full)
1999 latency_tolerant_lines = 1;
2000 else {
2001 if (lb_partitions <= (wm->vtaps + 1))
2002 latency_tolerant_lines = 1;
2003 else
2004 latency_tolerant_lines = 2;
2005 }
2006
2007 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2008
2009 if (dce6_latency_watermark(wm) <= latency_hiding)
2010 return true;
2011 else
2012 return false;
2013}
2014
2015static void dce6_program_watermarks(struct radeon_device *rdev,
2016 struct radeon_crtc *radeon_crtc,
2017 u32 lb_size, u32 num_heads)
2018{
2019 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2020 struct dce6_wm_params wm_low, wm_high;
2021 u32 dram_channels;
43b3cd99
AD		 2022 u32 pixel_period;
2023 u32 line_time = 0;
2024 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2025 u32 priority_a_mark = 0, priority_b_mark = 0;
2026 u32 priority_a_cnt = PRIORITY_OFF;
2027 u32 priority_b_cnt = PRIORITY_OFF;
2028 u32 tmp, arb_control3;
2029 fixed20_12 a, b, c;
2030
2031 if (radeon_crtc->base.enabled && num_heads && mode) {
2032 pixel_period = 1000000 / (u32)mode->clock;
2033 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2034 priority_a_cnt = 0;
2035 priority_b_cnt = 0;
2036
ca7db22b 2037 if (rdev->family == CHIP_ARUBA)
c696e53f 2038 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2039 else
c696e53f
AD		 2040 dram_channels = si_get_number_of_dram_channels(rdev);
2041
2042 /* watermark for high clocks */
2043 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2044 wm_high.yclk =
2045 radeon_dpm_get_mclk(rdev, false) * 10;
2046 wm_high.sclk =
2047 radeon_dpm_get_sclk(rdev, false) * 10;
2048 } else {
2049 wm_high.yclk = rdev->pm.current_mclk * 10;
2050 wm_high.sclk = rdev->pm.current_sclk * 10;
2051 }
2052
2053 wm_high.disp_clk = mode->clock;
2054 wm_high.src_width = mode->crtc_hdisplay;
2055 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2056 wm_high.blank_time = line_time - wm_high.active_time;
2057 wm_high.interlaced = false;
2058 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2059 wm_high.interlaced = true;
2060 wm_high.vsc = radeon_crtc->vsc;
2061 wm_high.vtaps = 1;
2062 if (radeon_crtc->rmx_type != RMX_OFF)
2063 wm_high.vtaps = 2;
2064 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2065 wm_high.lb_size = lb_size;
2066 wm_high.dram_channels = dram_channels;
2067 wm_high.num_heads = num_heads;
2068
2069 /* watermark for low clocks */
2070 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2071 wm_low.yclk =
2072 radeon_dpm_get_mclk(rdev, true) * 10;
2073 wm_low.sclk =
2074 radeon_dpm_get_sclk(rdev, true) * 10;
2075 } else {
2076 wm_low.yclk = rdev->pm.current_mclk * 10;
2077 wm_low.sclk = rdev->pm.current_sclk * 10;
2078 }
2079
2080 wm_low.disp_clk = mode->clock;
2081 wm_low.src_width = mode->crtc_hdisplay;
2082 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2083 wm_low.blank_time = line_time - wm_low.active_time;
2084 wm_low.interlaced = false;
2085 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2086 wm_low.interlaced = true;
2087 wm_low.vsc = radeon_crtc->vsc;
2088 wm_low.vtaps = 1;
2089 if (radeon_crtc->rmx_type != RMX_OFF)
2090 wm_low.vtaps = 2;
2091 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2092 wm_low.lb_size = lb_size;
2093 wm_low.dram_channels = dram_channels;
2094 wm_low.num_heads = num_heads;
43b3cd99
AD		 2095
2096 /* set for high clocks */
c696e53f 2097 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2098 /* set for low clocks */
c696e53f 2099 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2100
2101 /* possibly force display priority to high */
2102 /* should really do this at mode validation time... */
c696e53f
AD		 2103 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2104 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2105 !dce6_check_latency_hiding(&wm_high) ||
2106 (rdev->disp_priority == 2)) {
2107 DRM_DEBUG_KMS("force priority to high\n");
2108 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2109 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2110 }
2111 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2112 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2113 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2114 (rdev->disp_priority == 2)) {
2115 DRM_DEBUG_KMS("force priority to high\n");
2116 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2117 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2118 }
2119
2120 a.full = dfixed_const(1000);
2121 b.full = dfixed_const(mode->clock);
2122 b.full = dfixed_div(b, a);
2123 c.full = dfixed_const(latency_watermark_a);
2124 c.full = dfixed_mul(c, b);
2125 c.full = dfixed_mul(c, radeon_crtc->hsc);
2126 c.full = dfixed_div(c, a);
2127 a.full = dfixed_const(16);
2128 c.full = dfixed_div(c, a);
2129 priority_a_mark = dfixed_trunc(c);
2130 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2131
2132 a.full = dfixed_const(1000);
2133 b.full = dfixed_const(mode->clock);
2134 b.full = dfixed_div(b, a);
2135 c.full = dfixed_const(latency_watermark_b);
2136 c.full = dfixed_mul(c, b);
2137 c.full = dfixed_mul(c, radeon_crtc->hsc);
2138 c.full = dfixed_div(c, a);
2139 a.full = dfixed_const(16);
2140 c.full = dfixed_div(c, a);
2141 priority_b_mark = dfixed_trunc(c);
2142 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2143 }
2144
2145 /* select wm A */
2146 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2147 tmp = arb_control3;
2148 tmp &= ~LATENCY_WATERMARK_MASK(3);
2149 tmp |= LATENCY_WATERMARK_MASK(1);
2150 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2151 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2152 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2153 LATENCY_HIGH_WATERMARK(line_time)));
2154 /* select wm B */
2155 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2156 tmp &= ~LATENCY_WATERMARK_MASK(3);
2157 tmp |= LATENCY_WATERMARK_MASK(2);
2158 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2159 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2160 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2161 LATENCY_HIGH_WATERMARK(line_time)));
2162 /* restore original selection */
2163 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2164
2165 /* write the priority marks */
2166 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2167 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2168
2169}
2170
2171void dce6_bandwidth_update(struct radeon_device *rdev)
2172{
2173 struct drm_display_mode *mode0 = NULL;
2174 struct drm_display_mode *mode1 = NULL;
2175 u32 num_heads = 0, lb_size;
2176 int i;
2177
2178 radeon_update_display_priority(rdev);
2179
2180 for (i = 0; i < rdev->num_crtc; i++) {
2181 if (rdev->mode_info.crtcs[i]->base.enabled)
2182 num_heads++;
2183 }
2184 for (i = 0; i < rdev->num_crtc; i += 2) {
2185 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2186 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2187 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2188 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2189 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2190 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2191 }
2192}
2193
0a96d72b
AD		 2194/*
2195 * Core functions
2196 */
0a96d72b
AD		 2197static void si_tiling_mode_table_init(struct radeon_device *rdev)
2198{
2199 const u32 num_tile_mode_states = 32;
2200 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2201
2202 switch (rdev->config.si.mem_row_size_in_kb) {
2203 case 1:
2204 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2205 break;
2206 case 2:
2207 default:
2208 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2209 break;
2210 case 4:
2211 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2212 break;
2213 }
2214
2215 if ((rdev->family == CHIP_TAHITI) ||
2216 (rdev->family == CHIP_PITCAIRN)) {
2217 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2218 switch (reg_offset) {
2219 case 0: /* non-AA compressed depth or any compressed stencil */
2220 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2221 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2222 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2223 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2224 NUM_BANKS(ADDR_SURF_16_BANK) |
2225 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2226 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2227 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2228 break;
2229 case 1: /* 2xAA/4xAA compressed depth only */
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2234 NUM_BANKS(ADDR_SURF_16_BANK) |
2235 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2236 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2237 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2238 break;
2239 case 2: /* 8xAA compressed depth only */
2240 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2241 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2242 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2243 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2244 NUM_BANKS(ADDR_SURF_16_BANK) |
2245 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2246 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2247 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2248 break;
2249 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2250 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2251 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2252 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2253 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2254 NUM_BANKS(ADDR_SURF_16_BANK) |
2255 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2256 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2257 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2258 break;
2259 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2260 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2261 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2262 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2263 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2264 NUM_BANKS(ADDR_SURF_16_BANK) |
2265 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2266 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2267 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2268 break;
2269 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2270 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2271 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2272 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2273 TILE_SPLIT(split_equal_to_row_size) |
2274 NUM_BANKS(ADDR_SURF_16_BANK) |
2275 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2276 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2277 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2278 break;
2279 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2280 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2281 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2282 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2283 TILE_SPLIT(split_equal_to_row_size) |
2284 NUM_BANKS(ADDR_SURF_16_BANK) |
2285 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2286 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2287 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2288 break;
2289 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2291 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2292 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2293 TILE_SPLIT(split_equal_to_row_size) |
2294 NUM_BANKS(ADDR_SURF_16_BANK) |
2295 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2296 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2297 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2298 break;
2299 case 8: /* 1D and 1D Array Surfaces */
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2301 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2304 NUM_BANKS(ADDR_SURF_16_BANK) |
2305 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2306 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2307 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2308 break;
2309 case 9: /* Displayable maps. */
2310 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2311 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2312 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2313 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2314 NUM_BANKS(ADDR_SURF_16_BANK) |
2315 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2316 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2317 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2318 break;
2319 case 10: /* Display 8bpp. */
2320 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2322 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2323 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2324 NUM_BANKS(ADDR_SURF_16_BANK) |
2325 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2326 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2327 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2328 break;
2329 case 11: /* Display 16bpp. */
2330 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2331 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2332 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2333 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2334 NUM_BANKS(ADDR_SURF_16_BANK) |
2335 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2336 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2337 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2338 break;
2339 case 12: /* Display 32bpp. */
2340 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2341 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2342 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2343 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2344 NUM_BANKS(ADDR_SURF_16_BANK) |
2345 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2346 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2347 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2348 break;
2349 case 13: /* Thin. */
2350 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2351 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2352 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2353 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2354 NUM_BANKS(ADDR_SURF_16_BANK) |
2355 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2358 break;
2359 case 14: /* Thin 8 bpp. */
2360 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2361 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2362 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2363 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2364 NUM_BANKS(ADDR_SURF_16_BANK) |
2365 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2366 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2367 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2368 break;
2369 case 15: /* Thin 16 bpp. */
2370 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2371 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2372 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2373 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2374 NUM_BANKS(ADDR_SURF_16_BANK) |
2375 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2376 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2377 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2378 break;
2379 case 16: /* Thin 32 bpp. */
2380 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2381 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2382 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2383 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2384 NUM_BANKS(ADDR_SURF_16_BANK) |
2385 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2388 break;
2389 case 17: /* Thin 64 bpp. */
2390 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2391 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2392 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2393 TILE_SPLIT(split_equal_to_row_size) |
2394 NUM_BANKS(ADDR_SURF_16_BANK) |
2395 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2396 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2397 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2398 break;
2399 case 21: /* 8 bpp PRT. */
2400 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2401 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2402 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2403 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2404 NUM_BANKS(ADDR_SURF_16_BANK) |
2405 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2406 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2407 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2408 break;
2409 case 22: /* 16 bpp PRT */
2410 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2411 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2412 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2413 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2414 NUM_BANKS(ADDR_SURF_16_BANK) |
2415 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2418 break;
2419 case 23: /* 32 bpp PRT */
2420 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2421 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2422 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2423 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2424 NUM_BANKS(ADDR_SURF_16_BANK) |
2425 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2426 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2427 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2428 break;
2429 case 24: /* 64 bpp PRT */
2430 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2431 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2432 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2433 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2434 NUM_BANKS(ADDR_SURF_16_BANK) |
2435 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2436 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2437 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2438 break;
2439 case 25: /* 128 bpp PRT */
2440 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2441 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2442 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2443 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2444 NUM_BANKS(ADDR_SURF_8_BANK) |
2445 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2448 break;
2449 default:
2450 gb_tile_moden = 0;
2451 break;
2452 }
64d7b8be 2453 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2454 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2455 }
d0ae7fcc 2456 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2457 (rdev->family == CHIP_OLAND) ||
2458 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2459 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2460 switch (reg_offset) {
2461 case 0: /* non-AA compressed depth or any compressed stencil */
2462 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2463 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2464 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2465 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2466 NUM_BANKS(ADDR_SURF_16_BANK) |
2467 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2469 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2470 break;
2471 case 1: /* 2xAA/4xAA compressed depth only */
2472 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2473 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2474 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2475 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2476 NUM_BANKS(ADDR_SURF_16_BANK) |
2477 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2478 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2479 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2480 break;
2481 case 2: /* 8xAA compressed depth only */
2482 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2483 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2484 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2485 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2486 NUM_BANKS(ADDR_SURF_16_BANK) |
2487 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2490 break;
2491 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2492 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2493 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2494 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2495 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2496 NUM_BANKS(ADDR_SURF_16_BANK) |
2497 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2499 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2500 break;
2501 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2502 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2503 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2504 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2505 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2506 NUM_BANKS(ADDR_SURF_16_BANK) |
2507 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2508 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2509 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2510 break;
2511 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2512 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2513 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2514 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2515 TILE_SPLIT(split_equal_to_row_size) |
2516 NUM_BANKS(ADDR_SURF_16_BANK) |
2517 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2520 break;
2521 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2522 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2523 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2524 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2525 TILE_SPLIT(split_equal_to_row_size) |
2526 NUM_BANKS(ADDR_SURF_16_BANK) |
2527 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2528 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2529 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2530 break;
2531 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2532 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2533 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2534 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2535 TILE_SPLIT(split_equal_to_row_size) |
2536 NUM_BANKS(ADDR_SURF_16_BANK) |
2537 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2538 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2539 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2540 break;
2541 case 8: /* 1D and 1D Array Surfaces */
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2543 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2546 NUM_BANKS(ADDR_SURF_16_BANK) |
2547 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2548 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2549 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2550 break;
2551 case 9: /* Displayable maps. */
2552 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2553 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2554 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2555 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2556 NUM_BANKS(ADDR_SURF_16_BANK) |
2557 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2558 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2559 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2560 break;
2561 case 10: /* Display 8bpp. */
2562 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2563 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2564 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2565 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2566 NUM_BANKS(ADDR_SURF_16_BANK) |
2567 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2568 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2569 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2570 break;
2571 case 11: /* Display 16bpp. */
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2573 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2574 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2575 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2576 NUM_BANKS(ADDR_SURF_16_BANK) |
2577 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2578 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2579 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2580 break;
2581 case 12: /* Display 32bpp. */
2582 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2585 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2586 NUM_BANKS(ADDR_SURF_16_BANK) |
2587 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2588 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2589 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2590 break;
2591 case 13: /* Thin. */
2592 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2593 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2594 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2595 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2596 NUM_BANKS(ADDR_SURF_16_BANK) |
2597 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2598 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2599 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2600 break;
2601 case 14: /* Thin 8 bpp. */
2602 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2603 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2604 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2605 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2606 NUM_BANKS(ADDR_SURF_16_BANK) |
2607 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2609 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2610 break;
2611 case 15: /* Thin 16 bpp. */
2612 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2613 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2614 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2615 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2616 NUM_BANKS(ADDR_SURF_16_BANK) |
2617 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2618 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2619 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2620 break;
2621 case 16: /* Thin 32 bpp. */
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2623 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2624 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2625 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2626 NUM_BANKS(ADDR_SURF_16_BANK) |
2627 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2630 break;
2631 case 17: /* Thin 64 bpp. */
2632 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2633 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2634 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2635 TILE_SPLIT(split_equal_to_row_size) |
2636 NUM_BANKS(ADDR_SURF_16_BANK) |
2637 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2638 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2639 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2640 break;
2641 case 21: /* 8 bpp PRT. */
2642 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2643 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2644 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2645 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2646 NUM_BANKS(ADDR_SURF_16_BANK) |
2647 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2648 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2649 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2650 break;
2651 case 22: /* 16 bpp PRT */
2652 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2653 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2654 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2655 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2656 NUM_BANKS(ADDR_SURF_16_BANK) |
2657 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2658 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2659 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2660 break;
2661 case 23: /* 32 bpp PRT */
2662 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2663 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2664 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2665 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2666 NUM_BANKS(ADDR_SURF_16_BANK) |
2667 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2668 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2669 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2670 break;
2671 case 24: /* 64 bpp PRT */
2672 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2673 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2674 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2675 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2676 NUM_BANKS(ADDR_SURF_16_BANK) |
2677 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2678 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2679 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2680 break;
2681 case 25: /* 128 bpp PRT */
2682 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2683 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2684 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2685 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2686 NUM_BANKS(ADDR_SURF_8_BANK) |
2687 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2688 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2689 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2690 break;
2691 default:
2692 gb_tile_moden = 0;
2693 break;
2694 }
64d7b8be 2695 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2696 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2697 }
2698 } else
2699 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2700}
2701
1a8ca750
AD		 2702static void si_select_se_sh(struct radeon_device *rdev,
2703 u32 se_num, u32 sh_num)
2704{
2705 u32 data = INSTANCE_BROADCAST_WRITES;
2706
2707 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2708 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2709 else if (se_num == 0xffffffff)
2710 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2711 else if (sh_num == 0xffffffff)
2712 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2713 else
2714 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2715 WREG32(GRBM_GFX_INDEX, data);
2716}
2717
2718static u32 si_create_bitmask(u32 bit_width)
2719{
2720 u32 i, mask = 0;
2721
2722 for (i = 0; i < bit_width; i++) {
2723 mask <<= 1;
2724 mask |= 1;
2725 }
2726 return mask;
2727}
2728
2729static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2730{
2731 u32 data, mask;
2732
2733 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2734 if (data & 1)
2735 data &= INACTIVE_CUS_MASK;
2736 else
2737 data = 0;
2738 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2739
2740 data >>= INACTIVE_CUS_SHIFT;
2741
2742 mask = si_create_bitmask(cu_per_sh);
2743
2744 return ~data & mask;
2745}
2746
2747static void si_setup_spi(struct radeon_device *rdev,
2748 u32 se_num, u32 sh_per_se,
2749 u32 cu_per_sh)
2750{
2751 int i, j, k;
2752 u32 data, mask, active_cu;
2753
2754 for (i = 0; i < se_num; i++) {
2755 for (j = 0; j < sh_per_se; j++) {
2756 si_select_se_sh(rdev, i, j);
2757 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2758 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2759
2760 mask = 1;
2761 for (k = 0; k < 16; k++) {
2762 mask <<= k;
2763 if (active_cu & mask) {
2764 data &= ~mask;
2765 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2766 break;
2767 }
2768 }
2769 }
2770 }
2771 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2772}
2773
2774static u32 si_get_rb_disabled(struct radeon_device *rdev,
2775 u32 max_rb_num, u32 se_num,
2776 u32 sh_per_se)
2777{
2778 u32 data, mask;
2779
2780 data = RREG32(CC_RB_BACKEND_DISABLE);
2781 if (data & 1)
2782 data &= BACKEND_DISABLE_MASK;
2783 else
2784 data = 0;
2785 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2786
2787 data >>= BACKEND_DISABLE_SHIFT;
2788
2789 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2790
2791 return data & mask;
2792}
2793
2794static void si_setup_rb(struct radeon_device *rdev,
2795 u32 se_num, u32 sh_per_se,
2796 u32 max_rb_num)
2797{
2798 int i, j;
2799 u32 data, mask;
2800 u32 disabled_rbs = 0;
2801 u32 enabled_rbs = 0;
2802
2803 for (i = 0; i < se_num; i++) {
2804 for (j = 0; j < sh_per_se; j++) {
2805 si_select_se_sh(rdev, i, j);
2806 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2807 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2808 }
2809 }
2810 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2811
2812 mask = 1;
2813 for (i = 0; i < max_rb_num; i++) {
2814 if (!(disabled_rbs & mask))
2815 enabled_rbs |= mask;
2816 mask <<= 1;
2817 }
2818
2819 for (i = 0; i < se_num; i++) {
2820 si_select_se_sh(rdev, i, 0xffffffff);
2821 data = 0;
2822 for (j = 0; j < sh_per_se; j++) {
2823 switch (enabled_rbs & 3) {
2824 case 1:
2825 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2826 break;
2827 case 2:
2828 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2829 break;
2830 case 3:
2831 default:
2832 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2833 break;
2834 }
2835 enabled_rbs >>= 2;
2836 }
2837 WREG32(PA_SC_RASTER_CONFIG, data);
2838 }
2839 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2840}
2841
0a96d72b
AD		 2842static void si_gpu_init(struct radeon_device *rdev)
2843{
0a96d72b
AD		 2844 u32 gb_addr_config = 0;
2845 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2846 u32 sx_debug_1;
0a96d72b
AD		 2847 u32 hdp_host_path_cntl;
2848 u32 tmp;
2849 int i, j;
2850
2851 switch (rdev->family) {
2852 case CHIP_TAHITI:
2853 rdev->config.si.max_shader_engines = 2;
0a96d72b 2854 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2855 rdev->config.si.max_cu_per_sh = 8;
2856 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2857 rdev->config.si.max_backends_per_se = 4;
2858 rdev->config.si.max_texture_channel_caches = 12;
2859 rdev->config.si.max_gprs = 256;
2860 rdev->config.si.max_gs_threads = 32;
2861 rdev->config.si.max_hw_contexts = 8;
2862
2863 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2864 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2865 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2866 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2867 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2868 break;
2869 case CHIP_PITCAIRN:
2870 rdev->config.si.max_shader_engines = 2;
0a96d72b 2871 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2872 rdev->config.si.max_cu_per_sh = 5;
2873 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2874 rdev->config.si.max_backends_per_se = 4;
2875 rdev->config.si.max_texture_channel_caches = 8;
2876 rdev->config.si.max_gprs = 256;
2877 rdev->config.si.max_gs_threads = 32;
2878 rdev->config.si.max_hw_contexts = 8;
2879
2880 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2881 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2882 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2883 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2884 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2885 break;
2886 case CHIP_VERDE:
2887 default:
2888 rdev->config.si.max_shader_engines = 1;
0a96d72b 2889 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2890 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2891 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2892 rdev->config.si.max_backends_per_se = 4;
2893 rdev->config.si.max_texture_channel_caches = 4;
2894 rdev->config.si.max_gprs = 256;
2895 rdev->config.si.max_gs_threads = 32;
2896 rdev->config.si.max_hw_contexts = 8;
2897
d0ae7fcc
AD		 2898 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2899 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2900 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2901 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2902 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2903 break;
2904 case CHIP_OLAND:
2905 rdev->config.si.max_shader_engines = 1;
2906 rdev->config.si.max_tile_pipes = 4;
2907 rdev->config.si.max_cu_per_sh = 6;
2908 rdev->config.si.max_sh_per_se = 1;
2909 rdev->config.si.max_backends_per_se = 2;
2910 rdev->config.si.max_texture_channel_caches = 4;
2911 rdev->config.si.max_gprs = 256;
2912 rdev->config.si.max_gs_threads = 16;
2913 rdev->config.si.max_hw_contexts = 8;
2914
0a96d72b
AD		 2915 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2916 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2917 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2918 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2919 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2920 break;
8b02859d
AD		 2921 case CHIP_HAINAN:
2922 rdev->config.si.max_shader_engines = 1;
2923 rdev->config.si.max_tile_pipes = 4;
2924 rdev->config.si.max_cu_per_sh = 5;
2925 rdev->config.si.max_sh_per_se = 1;
2926 rdev->config.si.max_backends_per_se = 1;
2927 rdev->config.si.max_texture_channel_caches = 2;
2928 rdev->config.si.max_gprs = 256;
2929 rdev->config.si.max_gs_threads = 16;
2930 rdev->config.si.max_hw_contexts = 8;
2931
2932 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2933 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2934 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2935 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2936 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2937 break;
0a96d72b
AD		 2938 }
2939
2940 /* Initialize HDP */
2941 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2942 WREG32((0x2c14 + j), 0x00000000);
2943 WREG32((0x2c18 + j), 0x00000000);
2944 WREG32((0x2c1c + j), 0x00000000);
2945 WREG32((0x2c20 + j), 0x00000000);
2946 WREG32((0x2c24 + j), 0x00000000);
2947 }
2948
2949 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2950
2951 evergreen_fix_pci_max_read_req_size(rdev);
2952
2953 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2954
2955 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2956 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2957
0a96d72b 2958 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 2959 rdev->config.si.mem_max_burst_length_bytes = 256;
2960 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2961 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2962 if (rdev->config.si.mem_row_size_in_kb > 4)
2963 rdev->config.si.mem_row_size_in_kb = 4;
2964 /* XXX use MC settings? */
2965 rdev->config.si.shader_engine_tile_size = 32;
2966 rdev->config.si.num_gpus = 1;
2967 rdev->config.si.multi_gpu_tile_size = 64;
2968
1a8ca750
AD		 2969 /* fix up row size */
2970 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 2971 switch (rdev->config.si.mem_row_size_in_kb) {
2972 case 1:
2973 default:
2974 gb_addr_config |= ROW_SIZE(0);
2975 break;
2976 case 2:
2977 gb_addr_config |= ROW_SIZE(1);
2978 break;
2979 case 4:
2980 gb_addr_config |= ROW_SIZE(2);
2981 break;
2982 }
2983
0a96d72b
AD		 2984 /* setup tiling info dword. gb_addr_config is not adequate since it does
2985 * not have bank info, so create a custom tiling dword.
2986 * bits 3:0 num_pipes
2987 * bits 7:4 num_banks
2988 * bits 11:8 group_size
2989 * bits 15:12 row_size
2990 */
2991 rdev->config.si.tile_config = 0;
2992 switch (rdev->config.si.num_tile_pipes) {
2993 case 1:
2994 rdev->config.si.tile_config |= (0 << 0);
2995 break;
2996 case 2:
2997 rdev->config.si.tile_config |= (1 << 0);
2998 break;
2999 case 4:
3000 rdev->config.si.tile_config |= (2 << 0);
3001 break;
3002 case 8:
3003 default:
3004 /* XXX what about 12? */
3005 rdev->config.si.tile_config |= (3 << 0);
3006 break;
dca571a6
CK		 3007 }
3008 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3009 case 0: /* four banks */
1a8ca750 3010 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3011 break;
3012 case 1: /* eight banks */
3013 rdev->config.si.tile_config |= 1 << 4;
3014 break;
3015 case 2: /* sixteen banks */
3016 default:
3017 rdev->config.si.tile_config |= 2 << 4;
3018 break;
3019 }
0a96d72b
AD		 3020 rdev->config.si.tile_config |=
3021 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3022 rdev->config.si.tile_config |=
3023 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3024
0a96d72b
AD		 3025 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3026 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3027 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3028 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3029 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3030 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3031 if (rdev->has_uvd) {
3032 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3033 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3034 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3035 }
0a96d72b 3036
1a8ca750 3037 si_tiling_mode_table_init(rdev);
0a96d72b 3038
1a8ca750
AD		 3039 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3040 rdev->config.si.max_sh_per_se,
3041 rdev->config.si.max_backends_per_se);
0a96d72b 3042
1a8ca750
AD		 3043 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3044 rdev->config.si.max_sh_per_se,
3045 rdev->config.si.max_cu_per_sh);
0a96d72b 3046
0a96d72b
AD		 3047
3048 /* set HW defaults for 3D engine */
3049 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3050 ROQ_IB2_START(0x2b)));
3051 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3052
3053 sx_debug_1 = RREG32(SX_DEBUG_1);
3054 WREG32(SX_DEBUG_1, sx_debug_1);
3055
3056 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3057
3058 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3059 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3060 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3061 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3062
3063 WREG32(VGT_NUM_INSTANCES, 1);
3064
3065 WREG32(CP_PERFMON_CNTL, 0);
3066
3067 WREG32(SQ_CONFIG, 0);
3068
3069 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3070 FORCE_EOV_MAX_REZ_CNT(255)));
3071
3072 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3073 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3074
3075 WREG32(VGT_GS_VERTEX_REUSE, 16);
3076 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3077
3078 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3079 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3080 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3081 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3082 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3083 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3084 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3085 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3086
3087 tmp = RREG32(HDP_MISC_CNTL);
3088 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3089 WREG32(HDP_MISC_CNTL, tmp);
3090
3091 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3092 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3093
3094 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3095
3096 udelay(50);
3097}
c476dde2 3098
2ece2e8b
AD		 3099/*
3100 * GPU scratch registers helpers function.
3101 */
3102static void si_scratch_init(struct radeon_device *rdev)
3103{
3104 int i;
3105
3106 rdev->scratch.num_reg = 7;
3107 rdev->scratch.reg_base = SCRATCH_REG0;
3108 for (i = 0; i < rdev->scratch.num_reg; i++) {
3109 rdev->scratch.free[i] = true;
3110 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3111 }
3112}
3113
3114void si_fence_ring_emit(struct radeon_device *rdev,
3115 struct radeon_fence *fence)
3116{
3117 struct radeon_ring *ring = &rdev->ring[fence->ring];
3118 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3119
3120 /* flush read cache over gart */
3121 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3122 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3123 radeon_ring_write(ring, 0);
3124 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3125 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3126 PACKET3_TC_ACTION_ENA |
3127 PACKET3_SH_KCACHE_ACTION_ENA |
3128 PACKET3_SH_ICACHE_ACTION_ENA);
3129 radeon_ring_write(ring, 0xFFFFFFFF);
3130 radeon_ring_write(ring, 0);
3131 radeon_ring_write(ring, 10); /* poll interval */
3132 /* EVENT_WRITE_EOP - flush caches, send int */
3133 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3134 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3135 radeon_ring_write(ring, addr & 0xffffffff);
3136 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3137 radeon_ring_write(ring, fence->seq);
3138 radeon_ring_write(ring, 0);
3139}
3140
3141/*
3142 * IB stuff
3143 */
3144void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3145{
876dc9f3 3146 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 3147 u32 header;
3148
a85a7da4
AD		 3149 if (ib->is_const_ib) {
3150 /* set switch buffer packet before const IB */
3151 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3152 radeon_ring_write(ring, 0);
45df6803 3153
2ece2e8b 3154 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3155 } else {
89d35807 3156 u32 next_rptr;
a85a7da4 3157 if (ring->rptr_save_reg) {
89d35807 3158 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3159 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3160 radeon_ring_write(ring, ((ring->rptr_save_reg -
3161 PACKET3_SET_CONFIG_REG_START) >> 2));
3162 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3163 } else if (rdev->wb.enabled) {
3164 next_rptr = ring->wptr + 5 + 4 + 8;
3165 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3166 radeon_ring_write(ring, (1 << 8));
3167 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3168 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3169 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3170 }
3171
2ece2e8b 3172 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3173 }
2ece2e8b
AD		 3174
3175 radeon_ring_write(ring, header);
3176 radeon_ring_write(ring,
3177#ifdef __BIG_ENDIAN
3178 (2 << 0) |
3179#endif
3180 (ib->gpu_addr & 0xFFFFFFFC));
3181 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 3182 radeon_ring_write(ring, ib->length_dw |
3183 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 3184
a85a7da4
AD		 3185 if (!ib->is_const_ib) {
3186 /* flush read cache over gart for this vmid */
3187 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3188 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 3189 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 3190 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3191 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3192 PACKET3_TC_ACTION_ENA |
3193 PACKET3_SH_KCACHE_ACTION_ENA |
3194 PACKET3_SH_ICACHE_ACTION_ENA);
3195 radeon_ring_write(ring, 0xFFFFFFFF);
3196 radeon_ring_write(ring, 0);
3197 radeon_ring_write(ring, 10); /* poll interval */
3198 }
2ece2e8b
AD		 3199}
3200
48c0c902
AD		 3201/*
3202 * CP.
3203 */
3204static void si_cp_enable(struct radeon_device *rdev, bool enable)
3205{
3206 if (enable)
3207 WREG32(CP_ME_CNTL, 0);
3208 else {
3209 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3210 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3211 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3212 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3213 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3214 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3215 }
3216 udelay(50);
3217}
3218
3219static int si_cp_load_microcode(struct radeon_device *rdev)
3220{
3221 const __be32 *fw_data;
3222 int i;
3223
3224 if (!rdev->me_fw || !rdev->pfp_fw)
3225 return -EINVAL;
3226
3227 si_cp_enable(rdev, false);
3228
3229 /* PFP */
3230 fw_data = (const __be32 *)rdev->pfp_fw->data;
3231 WREG32(CP_PFP_UCODE_ADDR, 0);
3232 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3233 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3234 WREG32(CP_PFP_UCODE_ADDR, 0);
3235
3236 /* CE */
3237 fw_data = (const __be32 *)rdev->ce_fw->data;
3238 WREG32(CP_CE_UCODE_ADDR, 0);
3239 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3240 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3241 WREG32(CP_CE_UCODE_ADDR, 0);
3242
3243 /* ME */
3244 fw_data = (const __be32 *)rdev->me_fw->data;
3245 WREG32(CP_ME_RAM_WADDR, 0);
3246 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3247 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3248 WREG32(CP_ME_RAM_WADDR, 0);
3249
3250 WREG32(CP_PFP_UCODE_ADDR, 0);
3251 WREG32(CP_CE_UCODE_ADDR, 0);
3252 WREG32(CP_ME_RAM_WADDR, 0);
3253 WREG32(CP_ME_RAM_RADDR, 0);
3254 return 0;
3255}
3256
3257static int si_cp_start(struct radeon_device *rdev)
3258{
3259 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3260 int r, i;
3261
3262 r = radeon_ring_lock(rdev, ring, 7 + 4);
3263 if (r) {
3264 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3265 return r;
3266 }
3267 /* init the CP */
3268 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3269 radeon_ring_write(ring, 0x1);
3270 radeon_ring_write(ring, 0x0);
3271 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3272 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3273 radeon_ring_write(ring, 0);
3274 radeon_ring_write(ring, 0);
3275
3276 /* init the CE partitions */
3277 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3278 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3279 radeon_ring_write(ring, 0xc000);
3280 radeon_ring_write(ring, 0xe000);
3281 radeon_ring_unlock_commit(rdev, ring);
3282
3283 si_cp_enable(rdev, true);
3284
3285 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3286 if (r) {
3287 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3288 return r;
3289 }
3290
3291 /* setup clear context state */
3292 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3293 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3294
3295 for (i = 0; i < si_default_size; i++)
3296 radeon_ring_write(ring, si_default_state[i]);
3297
3298 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3299 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3300
3301 /* set clear context state */
3302 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3303 radeon_ring_write(ring, 0);
3304
3305 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3306 radeon_ring_write(ring, 0x00000316);
3307 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3308 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3309
3310 radeon_ring_unlock_commit(rdev, ring);
3311
3312 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3313 ring = &rdev->ring[i];
3314 r = radeon_ring_lock(rdev, ring, 2);
3315
3316 /* clear the compute context state */
3317 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3318 radeon_ring_write(ring, 0);
3319
3320 radeon_ring_unlock_commit(rdev, ring);
3321 }
3322
3323 return 0;
3324}
3325
3326static void si_cp_fini(struct radeon_device *rdev)
3327{
45df6803 3328 struct radeon_ring *ring;
48c0c902 3329 si_cp_enable(rdev, false);
45df6803
CK		 3330
3331 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3332 radeon_ring_fini(rdev, ring);
3333 radeon_scratch_free(rdev, ring->rptr_save_reg);
3334
3335 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3336 radeon_ring_fini(rdev, ring);
3337 radeon_scratch_free(rdev, ring->rptr_save_reg);
3338
3339 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3340 radeon_ring_fini(rdev, ring);
3341 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3342}
3343
3344static int si_cp_resume(struct radeon_device *rdev)
3345{
3346 struct radeon_ring *ring;
3347 u32 tmp;
3348 u32 rb_bufsz;
3349 int r;
3350
3351 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3352 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3353 SOFT_RESET_PA |
3354 SOFT_RESET_VGT |
3355 SOFT_RESET_SPI |
3356 SOFT_RESET_SX));
3357 RREG32(GRBM_SOFT_RESET);
3358 mdelay(15);
3359 WREG32(GRBM_SOFT_RESET, 0);
3360 RREG32(GRBM_SOFT_RESET);
3361
3362 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3363 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3364
3365 /* Set the write pointer delay */
3366 WREG32(CP_RB_WPTR_DELAY, 0);
3367
3368 WREG32(CP_DEBUG, 0);
3369 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3370
3371 /* ring 0 - compute and gfx */
3372 /* Set ring buffer size */
3373 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3374 rb_bufsz = drm_order(ring->ring_size / 8);
3375 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3376#ifdef __BIG_ENDIAN
3377 tmp |= BUF_SWAP_32BIT;
3378#endif
3379 WREG32(CP_RB0_CNTL, tmp);
3380
3381 /* Initialize the ring buffer's read and write pointers */
3382 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3383 ring->wptr = 0;
3384 WREG32(CP_RB0_WPTR, ring->wptr);
3385
48fc7f7e 3386 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3387 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3388 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3389
3390 if (rdev->wb.enabled)
3391 WREG32(SCRATCH_UMSK, 0xff);
3392 else {
3393 tmp |= RB_NO_UPDATE;
3394 WREG32(SCRATCH_UMSK, 0);
3395 }
3396
3397 mdelay(1);
3398 WREG32(CP_RB0_CNTL, tmp);
3399
3400 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3401
3402 ring->rptr = RREG32(CP_RB0_RPTR);
3403
3404 /* ring1 - compute only */
3405 /* Set ring buffer size */
3406 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3407 rb_bufsz = drm_order(ring->ring_size / 8);
3408 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3409#ifdef __BIG_ENDIAN
3410 tmp |= BUF_SWAP_32BIT;
3411#endif
3412 WREG32(CP_RB1_CNTL, tmp);
3413
3414 /* Initialize the ring buffer's read and write pointers */
3415 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3416 ring->wptr = 0;
3417 WREG32(CP_RB1_WPTR, ring->wptr);
3418
48fc7f7e 3419 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3420 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3421 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3422
3423 mdelay(1);
3424 WREG32(CP_RB1_CNTL, tmp);
3425
3426 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3427
3428 ring->rptr = RREG32(CP_RB1_RPTR);
3429
3430 /* ring2 - compute only */
3431 /* Set ring buffer size */
3432 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3433 rb_bufsz = drm_order(ring->ring_size / 8);
3434 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3435#ifdef __BIG_ENDIAN
3436 tmp |= BUF_SWAP_32BIT;
3437#endif
3438 WREG32(CP_RB2_CNTL, tmp);
3439
3440 /* Initialize the ring buffer's read and write pointers */
3441 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3442 ring->wptr = 0;
3443 WREG32(CP_RB2_WPTR, ring->wptr);
3444
48fc7f7e 3445 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3446 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3447 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3448
3449 mdelay(1);
3450 WREG32(CP_RB2_CNTL, tmp);
3451
3452 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3453
3454 ring->rptr = RREG32(CP_RB2_RPTR);
3455
3456 /* start the rings */
3457 si_cp_start(rdev);
3458 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3459 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3460 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3461 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3462 if (r) {
3463 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3464 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3465 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3466 return r;
3467 }
3468 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3469 if (r) {
3470 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3471 }
3472 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3473 if (r) {
3474 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3475 }
3476
3477 return 0;
3478}
3479
014bb209 3480static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3481{
014bb209 3482 u32 reset_mask = 0;
1c534671 3483 u32 tmp;
06bc6df0 3484
014bb209
AD		 3485 /* GRBM_STATUS */
3486 tmp = RREG32(GRBM_STATUS);
3487 if (tmp & (PA_BUSY | SC_BUSY |
3488 BCI_BUSY | SX_BUSY |
3489 TA_BUSY | VGT_BUSY |
3490 DB_BUSY | CB_BUSY |
3491 GDS_BUSY | SPI_BUSY |
3492 IA_BUSY | IA_BUSY_NO_DMA))
3493 reset_mask |= RADEON_RESET_GFX;
3494
3495 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3496 CP_BUSY | CP_COHERENCY_BUSY))
3497 reset_mask |= RADEON_RESET_CP;
3498
3499 if (tmp & GRBM_EE_BUSY)
3500 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3501
3502 /* GRBM_STATUS2 */
3503 tmp = RREG32(GRBM_STATUS2);
3504 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3505 reset_mask |= RADEON_RESET_RLC;
3506
3507 /* DMA_STATUS_REG 0 */
3508 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3509 if (!(tmp & DMA_IDLE))
3510 reset_mask |= RADEON_RESET_DMA;
3511
3512 /* DMA_STATUS_REG 1 */
3513 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3514 if (!(tmp & DMA_IDLE))
3515 reset_mask |= RADEON_RESET_DMA1;
3516
3517 /* SRBM_STATUS2 */
3518 tmp = RREG32(SRBM_STATUS2);
3519 if (tmp & DMA_BUSY)
3520 reset_mask |= RADEON_RESET_DMA;
3521
3522 if (tmp & DMA1_BUSY)
3523 reset_mask |= RADEON_RESET_DMA1;
3524
3525 /* SRBM_STATUS */
3526 tmp = RREG32(SRBM_STATUS);
3527
3528 if (tmp & IH_BUSY)
3529 reset_mask |= RADEON_RESET_IH;
3530
3531 if (tmp & SEM_BUSY)
3532 reset_mask |= RADEON_RESET_SEM;
3533
3534 if (tmp & GRBM_RQ_PENDING)
3535 reset_mask |= RADEON_RESET_GRBM;
3536
3537 if (tmp & VMC_BUSY)
3538 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3539
014bb209
AD		 3540 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3541 MCC_BUSY | MCD_BUSY))
3542 reset_mask |= RADEON_RESET_MC;
3543
3544 if (evergreen_is_display_hung(rdev))
3545 reset_mask |= RADEON_RESET_DISPLAY;
3546
3547 /* VM_L2_STATUS */
3548 tmp = RREG32(VM_L2_STATUS);
3549 if (tmp & L2_BUSY)
3550 reset_mask |= RADEON_RESET_VMC;
3551
d808fc88
AD		 3552 /* Skip MC reset as it's mostly likely not hung, just busy */
3553 if (reset_mask & RADEON_RESET_MC) {
3554 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3555 reset_mask &= ~RADEON_RESET_MC;
3556 }
3557
014bb209
AD		 3558 return reset_mask;
3559}
3560
3561static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3562{
3563 struct evergreen_mc_save save;
3564 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3565 u32 tmp;
19fc42ed 3566
06bc6df0 3567 if (reset_mask == 0)
014bb209 3568 return;
06bc6df0
AD		 3569
3570 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3571
1c534671 3572 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3573 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3574 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3575 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3576 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3577
1c534671
AD		 3578 /* Disable CP parsing/prefetching */
3579 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3580
3581 if (reset_mask & RADEON_RESET_DMA) {
3582 /* dma0 */
3583 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3584 tmp &= ~DMA_RB_ENABLE;
3585 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3586 }
3587 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3588 /* dma1 */
3589 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3590 tmp &= ~DMA_RB_ENABLE;
3591 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3592 }
3593
f770d78a
AD		 3594 udelay(50);
3595
3596 evergreen_mc_stop(rdev, &save);
3597 if (evergreen_mc_wait_for_idle(rdev)) {
3598 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3599 }
3600
1c534671
AD		 3601 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3602 grbm_soft_reset = SOFT_RESET_CB |
3603 SOFT_RESET_DB |
3604 SOFT_RESET_GDS |
3605 SOFT_RESET_PA |
3606 SOFT_RESET_SC |
3607 SOFT_RESET_BCI |
3608 SOFT_RESET_SPI |
3609 SOFT_RESET_SX |
3610 SOFT_RESET_TC |
3611 SOFT_RESET_TA |
3612 SOFT_RESET_VGT |
3613 SOFT_RESET_IA;
3614 }
3615
3616 if (reset_mask & RADEON_RESET_CP) {
3617 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3618
3619 srbm_soft_reset |= SOFT_RESET_GRBM;
3620 }
06bc6df0
AD		 3621
3622 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3623 srbm_soft_reset |= SOFT_RESET_DMA;
3624
3625 if (reset_mask & RADEON_RESET_DMA1)
3626 srbm_soft_reset |= SOFT_RESET_DMA1;
3627
3628 if (reset_mask & RADEON_RESET_DISPLAY)
3629 srbm_soft_reset |= SOFT_RESET_DC;
3630
3631 if (reset_mask & RADEON_RESET_RLC)
3632 grbm_soft_reset |= SOFT_RESET_RLC;
3633
3634 if (reset_mask & RADEON_RESET_SEM)
3635 srbm_soft_reset |= SOFT_RESET_SEM;
3636
3637 if (reset_mask & RADEON_RESET_IH)
3638 srbm_soft_reset |= SOFT_RESET_IH;
3639
3640 if (reset_mask & RADEON_RESET_GRBM)
3641 srbm_soft_reset |= SOFT_RESET_GRBM;
3642
3643 if (reset_mask & RADEON_RESET_VMC)
3644 srbm_soft_reset |= SOFT_RESET_VMC;
3645
3646 if (reset_mask & RADEON_RESET_MC)
3647 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3648
3649 if (grbm_soft_reset) {
3650 tmp = RREG32(GRBM_SOFT_RESET);
3651 tmp |= grbm_soft_reset;
3652 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3653 WREG32(GRBM_SOFT_RESET, tmp);
3654 tmp = RREG32(GRBM_SOFT_RESET);
3655
3656 udelay(50);
3657
3658 tmp &= ~grbm_soft_reset;
3659 WREG32(GRBM_SOFT_RESET, tmp);
3660 tmp = RREG32(GRBM_SOFT_RESET);
3661 }
3662
3663 if (srbm_soft_reset) {
3664 tmp = RREG32(SRBM_SOFT_RESET);
3665 tmp |= srbm_soft_reset;
3666 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3667 WREG32(SRBM_SOFT_RESET, tmp);
3668 tmp = RREG32(SRBM_SOFT_RESET);
3669
3670 udelay(50);
3671
3672 tmp &= ~srbm_soft_reset;
3673 WREG32(SRBM_SOFT_RESET, tmp);
3674 tmp = RREG32(SRBM_SOFT_RESET);
3675 }
06bc6df0
AD		 3676
3677 /* Wait a little for things to settle down */
3678 udelay(50);
3679
c476dde2 3680 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3681 udelay(50);
3682
1c534671 3683 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3684}
3685
3686int si_asic_reset(struct radeon_device *rdev)
3687{
014bb209
AD		 3688 u32 reset_mask;
3689
3690 reset_mask = si_gpu_check_soft_reset(rdev);
3691
3692 if (reset_mask)
3693 r600_set_bios_scratch_engine_hung(rdev, true);
3694
3695 si_gpu_soft_reset(rdev, reset_mask);
3696
3697 reset_mask = si_gpu_check_soft_reset(rdev);
3698
3699 if (!reset_mask)
3700 r600_set_bios_scratch_engine_hung(rdev, false);
3701
3702 return 0;
c476dde2
AD		 3703}
3704
123bc183
AD		 3705/**
3706 * si_gfx_is_lockup - Check if the GFX engine is locked up
3707 *
3708 * @rdev: radeon_device pointer
3709 * @ring: radeon_ring structure holding ring information
3710 *
3711 * Check if the GFX engine is locked up.
3712 * Returns true if the engine appears to be locked up, false if not.
3713 */
3714bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3715{
3716 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3717
3718 if (!(reset_mask & (RADEON_RESET_GFX |
3719 RADEON_RESET_COMPUTE |
3720 RADEON_RESET_CP))) {
3721 radeon_ring_lockup_update(ring);
3722 return false;
3723 }
3724 /* force CP activities */
3725 radeon_ring_force_activity(rdev, ring);
3726 return radeon_ring_test_lockup(rdev, ring);
3727}
3728
3729/**
3730 * si_dma_is_lockup - Check if the DMA engine is locked up
3731 *
3732 * @rdev: radeon_device pointer
3733 * @ring: radeon_ring structure holding ring information
3734 *
3735 * Check if the async DMA engine is locked up.
3736 * Returns true if the engine appears to be locked up, false if not.
3737 */
3738bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3739{
3740 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3741 u32 mask;
3742
3743 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3744 mask = RADEON_RESET_DMA;
3745 else
3746 mask = RADEON_RESET_DMA1;
3747
3748 if (!(reset_mask & mask)) {
3749 radeon_ring_lockup_update(ring);
3750 return false;
3751 }
3752 /* force ring activities */
3753 radeon_ring_force_activity(rdev, ring);
3754 return radeon_ring_test_lockup(rdev, ring);
3755}
3756
d2800ee5
AD		 3757/* MC */
3758static void si_mc_program(struct radeon_device *rdev)
3759{
3760 struct evergreen_mc_save save;
3761 u32 tmp;
3762 int i, j;
3763
3764 /* Initialize HDP */
3765 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3766 WREG32((0x2c14 + j), 0x00000000);
3767 WREG32((0x2c18 + j), 0x00000000);
3768 WREG32((0x2c1c + j), 0x00000000);
3769 WREG32((0x2c20 + j), 0x00000000);
3770 WREG32((0x2c24 + j), 0x00000000);
3771 }
3772 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3773
3774 evergreen_mc_stop(rdev, &save);
3775 if (radeon_mc_wait_for_idle(rdev)) {
3776 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3777 }
5153550a
AD		 3778 if (!ASIC_IS_NODCE(rdev))
3779 /* Lockout access through VGA aperture*/
3780 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3781 /* Update configuration */
3782 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3783 rdev->mc.vram_start >> 12);
3784 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3785 rdev->mc.vram_end >> 12);
3786 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3787 rdev->vram_scratch.gpu_addr >> 12);
3788 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3789 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3790 WREG32(MC_VM_FB_LOCATION, tmp);
3791 /* XXX double check these! */
3792 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3793 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3794 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3795 WREG32(MC_VM_AGP_BASE, 0);
3796 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3797 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3798 if (radeon_mc_wait_for_idle(rdev)) {
3799 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3800 }
3801 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3802 if (!ASIC_IS_NODCE(rdev)) {
3803 /* we need to own VRAM, so turn off the VGA renderer here
3804 * to stop it overwriting our objects */
3805 rv515_vga_render_disable(rdev);
3806 }
d2800ee5
AD		 3807}
3808
1c49165d
AD		 3809void si_vram_gtt_location(struct radeon_device *rdev,
3810 struct radeon_mc *mc)
d2800ee5
AD		 3811{
3812 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3813 /* leave room for at least 1024M GTT */
3814 dev_warn(rdev->dev, "limiting VRAM\n");
3815 mc->real_vram_size = 0xFFC0000000ULL;
3816 mc->mc_vram_size = 0xFFC0000000ULL;
3817 }
9ed8b1f9 3818 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3819 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3820 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3821}
3822
3823static int si_mc_init(struct radeon_device *rdev)
3824{
3825 u32 tmp;
3826 int chansize, numchan;
3827
3828 /* Get VRAM informations */
3829 rdev->mc.vram_is_ddr = true;
3830 tmp = RREG32(MC_ARB_RAMCFG);
3831 if (tmp & CHANSIZE_OVERRIDE) {
3832 chansize = 16;
3833 } else if (tmp & CHANSIZE_MASK) {
3834 chansize = 64;
3835 } else {
3836 chansize = 32;
3837 }
3838 tmp = RREG32(MC_SHARED_CHMAP);
3839 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3840 case 0:
3841 default:
3842 numchan = 1;
3843 break;
3844 case 1:
3845 numchan = 2;
3846 break;
3847 case 2:
3848 numchan = 4;
3849 break;
3850 case 3:
3851 numchan = 8;
3852 break;
3853 case 4:
3854 numchan = 3;
3855 break;
3856 case 5:
3857 numchan = 6;
3858 break;
3859 case 6:
3860 numchan = 10;
3861 break;
3862 case 7:
3863 numchan = 12;
3864 break;
3865 case 8:
3866 numchan = 16;
3867 break;
3868 }
3869 rdev->mc.vram_width = numchan * chansize;
3870 /* Could aper size report 0 ? */
3871 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3872 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3873 /* size in MB on si */
fc986034
NOS		 3874 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3875 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
d2800ee5
AD		 3876 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3877 si_vram_gtt_location(rdev, &rdev->mc);
3878 radeon_update_bandwidth_info(rdev);
3879
3880 return 0;
3881}
3882
3883/*
3884 * GART
3885 */
3886void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3887{
3888 /* flush hdp cache */
3889 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3890
3891 /* bits 0-15 are the VM contexts0-15 */
3892 WREG32(VM_INVALIDATE_REQUEST, 1);
3893}
3894
1109ca09 3895static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 3896{
3897 int r, i;
3898
3899 if (rdev->gart.robj == NULL) {
3900 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3901 return -EINVAL;
3902 }
3903 r = radeon_gart_table_vram_pin(rdev);
3904 if (r)
3905 return r;
3906 radeon_gart_restore(rdev);
3907 /* Setup TLB control */
3908 WREG32(MC_VM_MX_L1_TLB_CNTL,
3909 (0xA << 7) |
3910 ENABLE_L1_TLB |
3911 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3912 ENABLE_ADVANCED_DRIVER_MODEL |
3913 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3914 /* Setup L2 cache */
3915 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3916 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3917 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3918 EFFECTIVE_L2_QUEUE_SIZE(7) |
3919 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3920 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3921 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3922 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3923 /* setup context0 */
3924 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3925 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3926 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3927 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3928 (u32)(rdev->dummy_page.addr >> 12));
3929 WREG32(VM_CONTEXT0_CNTL2, 0);
3930 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3931 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3932
3933 WREG32(0x15D4, 0);
3934 WREG32(0x15D8, 0);
3935 WREG32(0x15DC, 0);
3936
3937 /* empty context1-15 */
d2800ee5
AD		 3938 /* set vm size, must be a multiple of 4 */
3939 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 3940 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 3941 /* Assign the pt base to something valid for now; the pts used for
3942 * the VMs are determined by the application and setup and assigned
3943 * on the fly in the vm part of radeon_gart.c
3944 */
d2800ee5
AD		 3945 for (i = 1; i < 16; i++) {
3946 if (i < 8)
3947 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3948 rdev->gart.table_addr >> 12);
3949 else
3950 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3951 rdev->gart.table_addr >> 12);
3952 }
3953
3954 /* enable context1-15 */
3955 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3956 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 3957 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 3958 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 3959 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3960 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3961 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3962 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3963 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3964 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3965 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3966 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3967 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3968 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3969 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3970 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 3971
3972 si_pcie_gart_tlb_flush(rdev);
3973 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3974 (unsigned)(rdev->mc.gtt_size >> 20),
3975 (unsigned long long)rdev->gart.table_addr);
3976 rdev->gart.ready = true;
3977 return 0;
3978}
3979
1109ca09 3980static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 3981{
3982 /* Disable all tables */
3983 WREG32(VM_CONTEXT0_CNTL, 0);
3984 WREG32(VM_CONTEXT1_CNTL, 0);
3985 /* Setup TLB control */
3986 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3987 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3988 /* Setup L2 cache */
3989 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3990 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3991 EFFECTIVE_L2_QUEUE_SIZE(7) |
3992 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3993 WREG32(VM_L2_CNTL2, 0);
3994 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3995 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3996 radeon_gart_table_vram_unpin(rdev);
3997}
3998
1109ca09 3999static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 4000{
4001 si_pcie_gart_disable(rdev);
4002 radeon_gart_table_vram_free(rdev);
4003 radeon_gart_fini(rdev);
4004}
4005
498dd8b3
AD		 4006/* vm parser */
4007static bool si_vm_reg_valid(u32 reg)
4008{
4009 /* context regs are fine */
4010 if (reg >= 0x28000)
4011 return true;
4012
4013 /* check config regs */
4014 switch (reg) {
4015 case GRBM_GFX_INDEX:
f418b88a 4016 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4017 case VGT_VTX_VECT_EJECT_REG:
4018 case VGT_CACHE_INVALIDATION:
4019 case VGT_ESGS_RING_SIZE:
4020 case VGT_GSVS_RING_SIZE:
4021 case VGT_GS_VERTEX_REUSE:
4022 case VGT_PRIMITIVE_TYPE:
4023 case VGT_INDEX_TYPE:
4024 case VGT_NUM_INDICES:
4025 case VGT_NUM_INSTANCES:
4026 case VGT_TF_RING_SIZE:
4027 case VGT_HS_OFFCHIP_PARAM:
4028 case VGT_TF_MEMORY_BASE:
4029 case PA_CL_ENHANCE:
4030 case PA_SU_LINE_STIPPLE_VALUE:
4031 case PA_SC_LINE_STIPPLE_STATE:
4032 case PA_SC_ENHANCE:
4033 case SQC_CACHES:
4034 case SPI_STATIC_THREAD_MGMT_1:
4035 case SPI_STATIC_THREAD_MGMT_2:
4036 case SPI_STATIC_THREAD_MGMT_3:
4037 case SPI_PS_MAX_WAVE_ID:
4038 case SPI_CONFIG_CNTL:
4039 case SPI_CONFIG_CNTL_1:
4040 case TA_CNTL_AUX:
4041 return true;
4042 default:
4043 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4044 return false;
4045 }
4046}
4047
4048static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4049 u32 *ib, struct radeon_cs_packet *pkt)
4050{
4051 switch (pkt->opcode) {
4052 case PACKET3_NOP:
4053 case PACKET3_SET_BASE:
4054 case PACKET3_SET_CE_DE_COUNTERS:
4055 case PACKET3_LOAD_CONST_RAM:
4056 case PACKET3_WRITE_CONST_RAM:
4057 case PACKET3_WRITE_CONST_RAM_OFFSET:
4058 case PACKET3_DUMP_CONST_RAM:
4059 case PACKET3_INCREMENT_CE_COUNTER:
4060 case PACKET3_WAIT_ON_DE_COUNTER:
4061 case PACKET3_CE_WRITE:
4062 break;
4063 default:
4064 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4065 return -EINVAL;
4066 }
4067 return 0;
4068}
4069
4070static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4071 u32 *ib, struct radeon_cs_packet *pkt)
4072{
4073 u32 idx = pkt->idx + 1;
4074 u32 idx_value = ib[idx];
4075 u32 start_reg, end_reg, reg, i;
5aa709be 4076 u32 command, info;
498dd8b3
AD		 4077
4078 switch (pkt->opcode) {
4079 case PACKET3_NOP:
4080 case PACKET3_SET_BASE:
4081 case PACKET3_CLEAR_STATE:
4082 case PACKET3_INDEX_BUFFER_SIZE:
4083 case PACKET3_DISPATCH_DIRECT:
4084 case PACKET3_DISPATCH_INDIRECT:
4085 case PACKET3_ALLOC_GDS:
4086 case PACKET3_WRITE_GDS_RAM:
4087 case PACKET3_ATOMIC_GDS:
4088 case PACKET3_ATOMIC:
4089 case PACKET3_OCCLUSION_QUERY:
4090 case PACKET3_SET_PREDICATION:
4091 case PACKET3_COND_EXEC:
4092 case PACKET3_PRED_EXEC:
4093 case PACKET3_DRAW_INDIRECT:
4094 case PACKET3_DRAW_INDEX_INDIRECT:
4095 case PACKET3_INDEX_BASE:
4096 case PACKET3_DRAW_INDEX_2:
4097 case PACKET3_CONTEXT_CONTROL:
4098 case PACKET3_INDEX_TYPE:
4099 case PACKET3_DRAW_INDIRECT_MULTI:
4100 case PACKET3_DRAW_INDEX_AUTO:
4101 case PACKET3_DRAW_INDEX_IMMD:
4102 case PACKET3_NUM_INSTANCES:
4103 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4104 case PACKET3_STRMOUT_BUFFER_UPDATE:
4105 case PACKET3_DRAW_INDEX_OFFSET_2:
4106 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4107 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4108 case PACKET3_MPEG_INDEX:
4109 case PACKET3_WAIT_REG_MEM:
4110 case PACKET3_MEM_WRITE:
4111 case PACKET3_PFP_SYNC_ME:
4112 case PACKET3_SURFACE_SYNC:
4113 case PACKET3_EVENT_WRITE:
4114 case PACKET3_EVENT_WRITE_EOP:
4115 case PACKET3_EVENT_WRITE_EOS:
4116 case PACKET3_SET_CONTEXT_REG:
4117 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4118 case PACKET3_SET_SH_REG:
4119 case PACKET3_SET_SH_REG_OFFSET:
4120 case PACKET3_INCREMENT_DE_COUNTER:
4121 case PACKET3_WAIT_ON_CE_COUNTER:
4122 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4123 case PACKET3_ME_WRITE:
4124 break;
4125 case PACKET3_COPY_DATA:
4126 if ((idx_value & 0xf00) == 0) {
4127 reg = ib[idx + 3] * 4;
4128 if (!si_vm_reg_valid(reg))
4129 return -EINVAL;
4130 }
4131 break;
4132 case PACKET3_WRITE_DATA:
4133 if ((idx_value & 0xf00) == 0) {
4134 start_reg = ib[idx + 1] * 4;
4135 if (idx_value & 0x10000) {
4136 if (!si_vm_reg_valid(start_reg))
4137 return -EINVAL;
4138 } else {
4139 for (i = 0; i < (pkt->count - 2); i++) {
4140 reg = start_reg + (4 * i);
4141 if (!si_vm_reg_valid(reg))
4142 return -EINVAL;
4143 }
4144 }
4145 }
4146 break;
4147 case PACKET3_COND_WRITE:
4148 if (idx_value & 0x100) {
4149 reg = ib[idx + 5] * 4;
4150 if (!si_vm_reg_valid(reg))
4151 return -EINVAL;
4152 }
4153 break;
4154 case PACKET3_COPY_DW:
4155 if (idx_value & 0x2) {
4156 reg = ib[idx + 3] * 4;
4157 if (!si_vm_reg_valid(reg))
4158 return -EINVAL;
4159 }
4160 break;
4161 case PACKET3_SET_CONFIG_REG:
4162 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4163 end_reg = 4 * pkt->count + start_reg - 4;
4164 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4165 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4166 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4167 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4168 return -EINVAL;
4169 }
4170 for (i = 0; i < pkt->count; i++) {
4171 reg = start_reg + (4 * i);
4172 if (!si_vm_reg_valid(reg))
4173 return -EINVAL;
4174 }
4175 break;
5aa709be
AD		 4176 case PACKET3_CP_DMA:
4177 command = ib[idx + 4];
4178 info = ib[idx + 1];
4179 if (command & PACKET3_CP_DMA_CMD_SAS) {
4180 /* src address space is register */
4181 if (((info & 0x60000000) >> 29) == 0) {
4182 start_reg = idx_value << 2;
4183 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4184 reg = start_reg;
4185 if (!si_vm_reg_valid(reg)) {
4186 DRM_ERROR("CP DMA Bad SRC register\n");
4187 return -EINVAL;
4188 }
4189 } else {
4190 for (i = 0; i < (command & 0x1fffff); i++) {
4191 reg = start_reg + (4 * i);
4192 if (!si_vm_reg_valid(reg)) {
4193 DRM_ERROR("CP DMA Bad SRC register\n");
4194 return -EINVAL;
4195 }
4196 }
4197 }
4198 }
4199 }
4200 if (command & PACKET3_CP_DMA_CMD_DAS) {
4201 /* dst address space is register */
4202 if (((info & 0x00300000) >> 20) == 0) {
4203 start_reg = ib[idx + 2];
4204 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4205 reg = start_reg;
4206 if (!si_vm_reg_valid(reg)) {
4207 DRM_ERROR("CP DMA Bad DST register\n");
4208 return -EINVAL;
4209 }
4210 } else {
4211 for (i = 0; i < (command & 0x1fffff); i++) {
4212 reg = start_reg + (4 * i);
4213 if (!si_vm_reg_valid(reg)) {
4214 DRM_ERROR("CP DMA Bad DST register\n");
4215 return -EINVAL;
4216 }
4217 }
4218 }
4219 }
4220 }
4221 break;
498dd8b3
AD		 4222 default:
4223 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4224 return -EINVAL;
4225 }
4226 return 0;
4227}
4228
4229static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4230 u32 *ib, struct radeon_cs_packet *pkt)
4231{
4232 u32 idx = pkt->idx + 1;
4233 u32 idx_value = ib[idx];
4234 u32 start_reg, reg, i;
4235
4236 switch (pkt->opcode) {
4237 case PACKET3_NOP:
4238 case PACKET3_SET_BASE:
4239 case PACKET3_CLEAR_STATE:
4240 case PACKET3_DISPATCH_DIRECT:
4241 case PACKET3_DISPATCH_INDIRECT:
4242 case PACKET3_ALLOC_GDS:
4243 case PACKET3_WRITE_GDS_RAM:
4244 case PACKET3_ATOMIC_GDS:
4245 case PACKET3_ATOMIC:
4246 case PACKET3_OCCLUSION_QUERY:
4247 case PACKET3_SET_PREDICATION:
4248 case PACKET3_COND_EXEC:
4249 case PACKET3_PRED_EXEC:
4250 case PACKET3_CONTEXT_CONTROL:
4251 case PACKET3_STRMOUT_BUFFER_UPDATE:
4252 case PACKET3_WAIT_REG_MEM:
4253 case PACKET3_MEM_WRITE:
4254 case PACKET3_PFP_SYNC_ME:
4255 case PACKET3_SURFACE_SYNC:
4256 case PACKET3_EVENT_WRITE:
4257 case PACKET3_EVENT_WRITE_EOP:
4258 case PACKET3_EVENT_WRITE_EOS:
4259 case PACKET3_SET_CONTEXT_REG:
4260 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4261 case PACKET3_SET_SH_REG:
4262 case PACKET3_SET_SH_REG_OFFSET:
4263 case PACKET3_INCREMENT_DE_COUNTER:
4264 case PACKET3_WAIT_ON_CE_COUNTER:
4265 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4266 case PACKET3_ME_WRITE:
4267 break;
4268 case PACKET3_COPY_DATA:
4269 if ((idx_value & 0xf00) == 0) {
4270 reg = ib[idx + 3] * 4;
4271 if (!si_vm_reg_valid(reg))
4272 return -EINVAL;
4273 }
4274 break;
4275 case PACKET3_WRITE_DATA:
4276 if ((idx_value & 0xf00) == 0) {
4277 start_reg = ib[idx + 1] * 4;
4278 if (idx_value & 0x10000) {
4279 if (!si_vm_reg_valid(start_reg))
4280 return -EINVAL;
4281 } else {
4282 for (i = 0; i < (pkt->count - 2); i++) {
4283 reg = start_reg + (4 * i);
4284 if (!si_vm_reg_valid(reg))
4285 return -EINVAL;
4286 }
4287 }
4288 }
4289 break;
4290 case PACKET3_COND_WRITE:
4291 if (idx_value & 0x100) {
4292 reg = ib[idx + 5] * 4;
4293 if (!si_vm_reg_valid(reg))
4294 return -EINVAL;
4295 }
4296 break;
4297 case PACKET3_COPY_DW:
4298 if (idx_value & 0x2) {
4299 reg = ib[idx + 3] * 4;
4300 if (!si_vm_reg_valid(reg))
4301 return -EINVAL;
4302 }
4303 break;
4304 default:
4305 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4306 return -EINVAL;
4307 }
4308 return 0;
4309}
4310
4311int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4312{
4313 int ret = 0;
4314 u32 idx = 0;
4315 struct radeon_cs_packet pkt;
4316
4317 do {
4318 pkt.idx = idx;
4e872ae2
IH		 4319 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4320 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4321 pkt.one_reg_wr = 0;
4322 switch (pkt.type) {
4e872ae2 4323 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4324 dev_err(rdev->dev, "Packet0 not allowed!\n");
4325 ret = -EINVAL;
4326 break;
4e872ae2 4327 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4328 idx += 1;
4329 break;
4e872ae2
IH		 4330 case RADEON_PACKET_TYPE3:
4331 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4332 if (ib->is_const_ib)
4333 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4334 else {
876dc9f3 4335 switch (ib->ring) {
498dd8b3
AD		 4336 case RADEON_RING_TYPE_GFX_INDEX:
4337 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4338 break;
4339 case CAYMAN_RING_TYPE_CP1_INDEX:
4340 case CAYMAN_RING_TYPE_CP2_INDEX:
4341 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4342 break;
4343 default:
876dc9f3 4344 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4345 ret = -EINVAL;
4346 break;
4347 }
4348 }
4349 idx += pkt.count + 2;
4350 break;
4351 default:
4352 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4353 ret = -EINVAL;
4354 break;
4355 }
4356 if (ret)
4357 break;
4358 } while (idx < ib->length_dw);
4359
4360 return ret;
4361}
4362
d2800ee5
AD		 4363/*
4364 * vm
4365 */
4366int si_vm_init(struct radeon_device *rdev)
4367{
4368 /* number of VMs */
4369 rdev->vm_manager.nvm = 16;
4370 /* base offset of vram pages */
4371 rdev->vm_manager.vram_base_offset = 0;
4372
4373 return 0;
4374}
4375
4376void si_vm_fini(struct radeon_device *rdev)
4377{
4378}
4379
82ffd92b
AD		 4380/**
4381 * si_vm_set_page - update the page tables using the CP
4382 *
4383 * @rdev: radeon_device pointer
43f1214a 4384 * @ib: indirect buffer to fill with commands
82ffd92b
AD		 4385 * @pe: addr of the page entry
4386 * @addr: dst addr to write into pe
4387 * @count: number of page entries to update
4388 * @incr: increase next addr by incr bytes
4389 * @flags: access flags
4390 *
43f1214a 4391 * Update the page tables using the CP (SI).
82ffd92b 4392 */
43f1214a
AD		 4393void si_vm_set_page(struct radeon_device *rdev,
4394 struct radeon_ib *ib,
4395 uint64_t pe,
82ffd92b
AD		 4396 uint64_t addr, unsigned count,
4397 uint32_t incr, uint32_t flags)
d2800ee5 4398{
82ffd92b 4399 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
deab48f1
AD		 4400 uint64_t value;
4401 unsigned ndw;
4402
4403 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4404 while (count) {
4405 ndw = 2 + count * 2;
4406 if (ndw > 0x3FFE)
4407 ndw = 0x3FFE;
4408
43f1214a
AD		 4409 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4410 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4411 WRITE_DATA_DST_SEL(1));
4412 ib->ptr[ib->length_dw++] = pe;
4413 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
deab48f1
AD		 4414 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4415 if (flags & RADEON_VM_PAGE_SYSTEM) {
4416 value = radeon_vm_map_gart(rdev, addr);
4417 value &= 0xFFFFFFFFFFFFF000ULL;
4418 } else if (flags & RADEON_VM_PAGE_VALID) {
4419 value = addr;
4420 } else {
4421 value = 0;
4422 }
4423 addr += incr;
4424 value |= r600_flags;
43f1214a
AD		 4425 ib->ptr[ib->length_dw++] = value;
4426 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4427 }
4428 }
4429 } else {
4430 /* DMA */
4431 if (flags & RADEON_VM_PAGE_SYSTEM) {
4432 while (count) {
4433 ndw = count * 2;
4434 if (ndw > 0xFFFFE)
4435 ndw = 0xFFFFE;
4436
4437 /* for non-physically contiguous pages (system) */
43f1214a
AD		 4438 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
4439 ib->ptr[ib->length_dw++] = pe;
4440 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
deab48f1
AD		 4441 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
4442 if (flags & RADEON_VM_PAGE_SYSTEM) {
4443 value = radeon_vm_map_gart(rdev, addr);
4444 value &= 0xFFFFFFFFFFFFF000ULL;
4445 } else if (flags & RADEON_VM_PAGE_VALID) {
4446 value = addr;
4447 } else {
4448 value = 0;
4449 }
4450 addr += incr;
4451 value |= r600_flags;
43f1214a
AD		 4452 ib->ptr[ib->length_dw++] = value;
4453 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4454 }
4455 }
4456 } else {
4457 while (count) {
4458 ndw = count * 2;
4459 if (ndw > 0xFFFFE)
4460 ndw = 0xFFFFE;
4461
4462 if (flags & RADEON_VM_PAGE_VALID)
4463 value = addr;
4464 else
4465 value = 0;
4466 /* for physically contiguous pages (vram) */
43f1214a
AD		 4467 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
4468 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4469 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4470 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4471 ib->ptr[ib->length_dw++] = 0;
4472 ib->ptr[ib->length_dw++] = value; /* value */
4473 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4474 ib->ptr[ib->length_dw++] = incr; /* increment size */
4475 ib->ptr[ib->length_dw++] = 0;
deab48f1
AD		 4476 pe += ndw * 4;
4477 addr += (ndw / 2) * incr;
4478 count -= ndw / 2;
4479 }
d7025d89 4480 }
43f1214a
AD		 4481 while (ib->length_dw & 0x7)
4482 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
82ffd92b 4483 }
d2800ee5
AD		 4484}
4485
498522b4 4486void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4487{
498522b4 4488 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4489
ee60e29f 4490 if (vm == NULL)
d2800ee5
AD		 4491 return;
4492
76c44f2c
AD		 4493 /* write new base address */
4494 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4495 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4496 WRITE_DATA_DST_SEL(0)));
4497
ee60e29f 4498 if (vm->id < 8) {
76c44f2c
AD		 4499 radeon_ring_write(ring,
4500 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4501 } else {
76c44f2c
AD		 4502 radeon_ring_write(ring,
4503 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4504 }
76c44f2c 4505 radeon_ring_write(ring, 0);
fa87e62d 4506 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4507
d2800ee5 4508 /* flush hdp cache */
76c44f2c
AD		 4509 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4510 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4511 WRITE_DATA_DST_SEL(0)));
4512 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4513 radeon_ring_write(ring, 0);
ee60e29f
CK		 4514 radeon_ring_write(ring, 0x1);
4515
d2800ee5 4516 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4517 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4518 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4519 WRITE_DATA_DST_SEL(0)));
4520 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4521 radeon_ring_write(ring, 0);
498522b4 4522 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4523
4524 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4525 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4526 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4527}
4528
8c5fd7ef
AD		 4529void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4530{
4531 struct radeon_ring *ring = &rdev->ring[ridx];
4532
4533 if (vm == NULL)
4534 return;
4535
4536 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4537 if (vm->id < 8) {
4538 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4539 } else {
4540 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4541 }
4542 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4543
4544 /* flush hdp cache */
4545 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4546 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4547 radeon_ring_write(ring, 1);
4548
4549 /* bits 0-7 are the VM contexts0-7 */
4550 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4551 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4552 radeon_ring_write(ring, 1 << vm->id);
4553}
4554
347e7592
AD		 4555/*
4556 * RLC
4557 */
c420c745 4558void si_rlc_fini(struct radeon_device *rdev)
347e7592
AD		 4559{
4560 int r;
4561
4562 /* save restore block */
4563 if (rdev->rlc.save_restore_obj) {
4564 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4565 if (unlikely(r != 0))
4566 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4567 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4568 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4569
4570 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4571 rdev->rlc.save_restore_obj = NULL;
4572 }
4573
4574 /* clear state block */
4575 if (rdev->rlc.clear_state_obj) {
4576 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4577 if (unlikely(r != 0))
4578 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4579 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4580 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4581
4582 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4583 rdev->rlc.clear_state_obj = NULL;
4584 }
4585}
4586
bd8cd539
AD		 4587#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4588
c420c745 4589int si_rlc_init(struct radeon_device *rdev)
347e7592 4590{
6d8cf000 4591 volatile u32 *dst_ptr;
bd8cd539
AD		 4592 u32 dws, data, i, j, k, reg_num;
4593 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
4594 u64 reg_list_mc_addr;
4595 const struct cs_section_def *cs_data = si_cs_data;
4596 int r;
347e7592
AD		 4597
4598 /* save restore block */
4599 if (rdev->rlc.save_restore_obj == NULL) {
4600 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
40f5cf99
AD		 4601 RADEON_GEM_DOMAIN_VRAM, NULL,
4602 &rdev->rlc.save_restore_obj);
347e7592
AD		 4603 if (r) {
4604 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4605 return r;
4606 }
4607 }
4608
4609 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4610 if (unlikely(r != 0)) {
4611 si_rlc_fini(rdev);
4612 return r;
4613 }
4614 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4615 &rdev->rlc.save_restore_gpu_addr);
4616 if (r) {
6d8cf000 4617 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
347e7592
AD		 4618 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4619 si_rlc_fini(rdev);
4620 return r;
4621 }
4622
6d8cf000
AD		 4623 if (rdev->family == CHIP_VERDE) {
4624 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4625 if (r) {
4626 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4627 si_rlc_fini(rdev);
4628 return r;
4629 }
4630 /* write the sr buffer */
4631 dst_ptr = rdev->rlc.sr_ptr;
4632 for (i = 0; i < ARRAY_SIZE(verde_rlc_save_restore_register_list); i++) {
4633 dst_ptr[i] = verde_rlc_save_restore_register_list[i];
4634 }
4635 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4636 }
4637 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4638
347e7592 4639 /* clear state block */
bd8cd539
AD		 4640 reg_list_num = 0;
4641 dws = 0;
4642 for (i = 0; cs_data[i].section != NULL; i++) {
4643 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4644 reg_list_num++;
4645 dws += cs_data[i].section[j].reg_count;
4646 }
4647 }
4648 reg_list_blk_index = (3 * reg_list_num + 2);
4649 dws += reg_list_blk_index;
4650
347e7592 4651 if (rdev->rlc.clear_state_obj == NULL) {
bd8cd539
AD		 4652 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4653 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
347e7592
AD		 4654 if (r) {
4655 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4656 si_rlc_fini(rdev);
4657 return r;
4658 }
4659 }
4660 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4661 if (unlikely(r != 0)) {
4662 si_rlc_fini(rdev);
4663 return r;
4664 }
4665 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4666 &rdev->rlc.clear_state_gpu_addr);
4667 if (r) {
bd8cd539
AD		 4668
4669 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
347e7592
AD		 4670 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4671 si_rlc_fini(rdev);
4672 return r;
4673 }
bd8cd539
AD		 4674 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4675 if (r) {
4676 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4677 si_rlc_fini(rdev);
4678 return r;
4679 }
4680 /* set up the cs buffer */
4681 dst_ptr = rdev->rlc.cs_ptr;
4682 reg_list_hdr_blk_index = 0;
4683 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4684 data = upper_32_bits(reg_list_mc_addr);
4685 dst_ptr[reg_list_hdr_blk_index] = data;
4686 reg_list_hdr_blk_index++;
4687 for (i = 0; cs_data[i].section != NULL; i++) {
4688 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4689 reg_num = cs_data[i].section[j].reg_count;
4690 data = reg_list_mc_addr & 0xffffffff;
4691 dst_ptr[reg_list_hdr_blk_index] = data;
4692 reg_list_hdr_blk_index++;
4693
4694 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4695 dst_ptr[reg_list_hdr_blk_index] = data;
4696 reg_list_hdr_blk_index++;
4697
4698 data = 0x08000000 | (reg_num * 4);
4699 dst_ptr[reg_list_hdr_blk_index] = data;
4700 reg_list_hdr_blk_index++;
4701
4702 for (k = 0; k < reg_num; k++) {
4703 data = cs_data[i].section[j].extent[k];
4704 dst_ptr[reg_list_blk_index + k] = data;
4705 }
4706 reg_list_mc_addr += reg_num * 4;
4707 reg_list_blk_index += reg_num;
4708 }
4709 }
4710 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4711
4712 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4713 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
347e7592
AD		 4714
4715 return 0;
4716}
4717
d719cef3
AD		 4718static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4719 bool enable)
4720{
4721 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4722 u32 mask;
4723 int i;
4724
4725 if (enable)
4726 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4727 else
4728 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4729 WREG32(CP_INT_CNTL_RING0, tmp);
4730
4731 if (!enable) {
4732 /* read a gfx register */
4733 tmp = RREG32(DB_DEPTH_INFO);
4734
4735 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4736 for (i = 0; i < rdev->usec_timeout; i++) {
4737 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4738 break;
4739 udelay(1);
4740 }
4741 }
4742}
4743
4744static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4745{
4746 int i;
4747
4748 for (i = 0; i < rdev->usec_timeout; i++) {
4749 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4750 break;
4751 udelay(1);
4752 }
4753
4754 for (i = 0; i < rdev->usec_timeout; i++) {
4755 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4756 break;
4757 udelay(1);
4758 }
4759}
4760
347e7592
AD		 4761static void si_rlc_stop(struct radeon_device *rdev)
4762{
4763 WREG32(RLC_CNTL, 0);
d719cef3
AD		 4764
4765 si_enable_gui_idle_interrupt(rdev, false);
4766
4767 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 4768}
4769
4770static void si_rlc_start(struct radeon_device *rdev)
4771{
4772 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 4773
4774 si_enable_gui_idle_interrupt(rdev, true);
4775
4776 udelay(50);
4777}
4778
4779static bool si_lbpw_supported(struct radeon_device *rdev)
4780{
4781 u32 tmp;
4782
4783 /* Enable LBPW only for DDR3 */
4784 tmp = RREG32(MC_SEQ_MISC0);
4785 if ((tmp & 0xF0000000) == 0xB0000000)
4786 return true;
4787 return false;
4788}
4789
4790static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
4791{
4792 u32 tmp;
4793
4794 tmp = RREG32(RLC_LB_CNTL);
4795 if (enable)
4796 tmp |= LOAD_BALANCE_ENABLE;
4797 else
4798 tmp &= ~LOAD_BALANCE_ENABLE;
4799 WREG32(RLC_LB_CNTL, tmp);
4800
4801 if (!enable) {
4802 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4803 WREG32(SPI_LB_CU_MASK, 0x00ff);
4804 }
347e7592
AD		 4805}
4806
4807static int si_rlc_resume(struct radeon_device *rdev)
4808{
4809 u32 i;
4810 const __be32 *fw_data;
4811
4812 if (!rdev->rlc_fw)
4813 return -EINVAL;
4814
4815 si_rlc_stop(rdev);
4816
4817 WREG32(RLC_RL_BASE, 0);
4818 WREG32(RLC_RL_SIZE, 0);
4819 WREG32(RLC_LB_CNTL, 0);
4820 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
4821 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 4822 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592
AD		 4823
4824 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4825 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4826
4827 WREG32(RLC_MC_CNTL, 0);
4828 WREG32(RLC_UCODE_CNTL, 0);
4829
4830 fw_data = (const __be32 *)rdev->rlc_fw->data;
4831 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
4832 WREG32(RLC_UCODE_ADDR, i);
4833 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4834 }
4835 WREG32(RLC_UCODE_ADDR, 0);
4836
d719cef3
AD		 4837 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
4838
347e7592
AD		 4839 si_rlc_start(rdev);
4840
4841 return 0;
4842}
4843
25a857fb
AD		 4844static void si_enable_interrupts(struct radeon_device *rdev)
4845{
4846 u32 ih_cntl = RREG32(IH_CNTL);
4847 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4848
4849 ih_cntl |= ENABLE_INTR;
4850 ih_rb_cntl |= IH_RB_ENABLE;
4851 WREG32(IH_CNTL, ih_cntl);
4852 WREG32(IH_RB_CNTL, ih_rb_cntl);
4853 rdev->ih.enabled = true;
4854}
4855
4856static void si_disable_interrupts(struct radeon_device *rdev)
4857{
4858 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4859 u32 ih_cntl = RREG32(IH_CNTL);
4860
4861 ih_rb_cntl &= ~IH_RB_ENABLE;
4862 ih_cntl &= ~ENABLE_INTR;
4863 WREG32(IH_RB_CNTL, ih_rb_cntl);
4864 WREG32(IH_CNTL, ih_cntl);
4865 /* set rptr, wptr to 0 */
4866 WREG32(IH_RB_RPTR, 0);
4867 WREG32(IH_RB_WPTR, 0);
4868 rdev->ih.enabled = false;
25a857fb
AD		 4869 rdev->ih.rptr = 0;
4870}
4871
4872static void si_disable_interrupt_state(struct radeon_device *rdev)
4873{
4874 u32 tmp;
4875
4876 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4877 WREG32(CP_INT_CNTL_RING1, 0);
4878 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 4879 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4880 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
4881 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4882 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 4883 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 4884 if (rdev->num_crtc >= 2) {
4885 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4886 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4887 }
25a857fb
AD		 4888 if (rdev->num_crtc >= 4) {
4889 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4890 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4891 }
4892 if (rdev->num_crtc >= 6) {
4893 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4894 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4895 }
4896
5153550a
AD		 4897 if (rdev->num_crtc >= 2) {
4898 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4899 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4900 }
25a857fb
AD		 4901 if (rdev->num_crtc >= 4) {
4902 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4903 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4904 }
4905 if (rdev->num_crtc >= 6) {
4906 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4907 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4908 }
4909
5153550a
AD		 4910 if (!ASIC_IS_NODCE(rdev)) {
4911 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4912
4913 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4914 WREG32(DC_HPD1_INT_CONTROL, tmp);
4915 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4916 WREG32(DC_HPD2_INT_CONTROL, tmp);
4917 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4918 WREG32(DC_HPD3_INT_CONTROL, tmp);
4919 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4920 WREG32(DC_HPD4_INT_CONTROL, tmp);
4921 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4922 WREG32(DC_HPD5_INT_CONTROL, tmp);
4923 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4924 WREG32(DC_HPD6_INT_CONTROL, tmp);
4925 }
25a857fb
AD		 4926}
4927
4928static int si_irq_init(struct radeon_device *rdev)
4929{
4930 int ret = 0;
4931 int rb_bufsz;
4932 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
4933
4934 /* allocate ring */
4935 ret = r600_ih_ring_alloc(rdev);
4936 if (ret)
4937 return ret;
4938
4939 /* disable irqs */
4940 si_disable_interrupts(rdev);
4941
4942 /* init rlc */
4943 ret = si_rlc_resume(rdev);
4944 if (ret) {
4945 r600_ih_ring_fini(rdev);
4946 return ret;
4947 }
4948
4949 /* setup interrupt control */
4950 /* set dummy read address to ring address */
4951 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
4952 interrupt_cntl = RREG32(INTERRUPT_CNTL);
4953 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
4954 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
4955 */
4956 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
4957 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
4958 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
4959 WREG32(INTERRUPT_CNTL, interrupt_cntl);
4960
4961 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
4962 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
4963
4964 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
4965 IH_WPTR_OVERFLOW_CLEAR |
4966 (rb_bufsz << 1));
4967
4968 if (rdev->wb.enabled)
4969 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
4970
4971 /* set the writeback address whether it's enabled or not */
4972 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
4973 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
4974
4975 WREG32(IH_RB_CNTL, ih_rb_cntl);
4976
4977 /* set rptr, wptr to 0 */
4978 WREG32(IH_RB_RPTR, 0);
4979 WREG32(IH_RB_WPTR, 0);
4980
4981 /* Default settings for IH_CNTL (disabled at first) */
4982 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
4983 /* RPTR_REARM only works if msi's are enabled */
4984 if (rdev->msi_enabled)
4985 ih_cntl |= RPTR_REARM;
4986 WREG32(IH_CNTL, ih_cntl);
4987
4988 /* force the active interrupt state to all disabled */
4989 si_disable_interrupt_state(rdev);
4990
2099810f
DA		 4991 pci_set_master(rdev->pdev);
4992
25a857fb
AD		 4993 /* enable irqs */
4994 si_enable_interrupts(rdev);
4995
4996 return ret;
4997}
4998
4999int si_irq_set(struct radeon_device *rdev)
5000{
5001 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
5002 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5003 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 5004 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 5005 u32 grbm_int_cntl = 0;
5006 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 5007 u32 dma_cntl, dma_cntl1;
25a857fb
AD		 5008
5009 if (!rdev->irq.installed) {
5010 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5011 return -EINVAL;
5012 }
5013 /* don't enable anything if the ih is disabled */
5014 if (!rdev->ih.enabled) {
5015 si_disable_interrupts(rdev);
5016 /* force the active interrupt state to all disabled */
5017 si_disable_interrupt_state(rdev);
5018 return 0;
5019 }
5020
5153550a
AD		 5021 if (!ASIC_IS_NODCE(rdev)) {
5022 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5023 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5024 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5025 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5026 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5027 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5028 }
25a857fb 5029
8c5fd7ef
AD		 5030 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5031 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5032
25a857fb 5033 /* enable CP interrupts on all rings */
736fc37f 5034 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 5035 DRM_DEBUG("si_irq_set: sw int gfx\n");
5036 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5037 }
736fc37f 5038 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 5039 DRM_DEBUG("si_irq_set: sw int cp1\n");
5040 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5041 }
736fc37f 5042 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 5043 DRM_DEBUG("si_irq_set: sw int cp2\n");
5044 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5045 }
8c5fd7ef
AD		 5046 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5047 DRM_DEBUG("si_irq_set: sw int dma\n");
5048 dma_cntl |= TRAP_ENABLE;
5049 }
5050
5051 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5052 DRM_DEBUG("si_irq_set: sw int dma1\n");
5053 dma_cntl1 |= TRAP_ENABLE;
5054 }
25a857fb 5055 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 5056 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 5057 DRM_DEBUG("si_irq_set: vblank 0\n");
5058 crtc1 |= VBLANK_INT_MASK;
5059 }
5060 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 5061 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 5062 DRM_DEBUG("si_irq_set: vblank 1\n");
5063 crtc2 |= VBLANK_INT_MASK;
5064 }
5065 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 5066 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 5067 DRM_DEBUG("si_irq_set: vblank 2\n");
5068 crtc3 |= VBLANK_INT_MASK;
5069 }
5070 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 5071 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 5072 DRM_DEBUG("si_irq_set: vblank 3\n");
5073 crtc4 |= VBLANK_INT_MASK;
5074 }
5075 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 5076 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 5077 DRM_DEBUG("si_irq_set: vblank 4\n");
5078 crtc5 |= VBLANK_INT_MASK;
5079 }
5080 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 5081 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 5082 DRM_DEBUG("si_irq_set: vblank 5\n");
5083 crtc6 |= VBLANK_INT_MASK;
5084 }
5085 if (rdev->irq.hpd[0]) {
5086 DRM_DEBUG("si_irq_set: hpd 1\n");
5087 hpd1 |= DC_HPDx_INT_EN;
5088 }
5089 if (rdev->irq.hpd[1]) {
5090 DRM_DEBUG("si_irq_set: hpd 2\n");
5091 hpd2 |= DC_HPDx_INT_EN;
5092 }
5093 if (rdev->irq.hpd[2]) {
5094 DRM_DEBUG("si_irq_set: hpd 3\n");
5095 hpd3 |= DC_HPDx_INT_EN;
5096 }
5097 if (rdev->irq.hpd[3]) {
5098 DRM_DEBUG("si_irq_set: hpd 4\n");
5099 hpd4 |= DC_HPDx_INT_EN;
5100 }
5101 if (rdev->irq.hpd[4]) {
5102 DRM_DEBUG("si_irq_set: hpd 5\n");
5103 hpd5 |= DC_HPDx_INT_EN;
5104 }
5105 if (rdev->irq.hpd[5]) {
5106 DRM_DEBUG("si_irq_set: hpd 6\n");
5107 hpd6 |= DC_HPDx_INT_EN;
5108 }
25a857fb
AD		 5109
5110 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5111 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5112 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5113
8c5fd7ef
AD		 5114 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5115 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5116
25a857fb
AD		 5117 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5118
5153550a
AD		 5119 if (rdev->num_crtc >= 2) {
5120 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5121 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5122 }
25a857fb
AD		 5123 if (rdev->num_crtc >= 4) {
5124 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5125 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5126 }
5127 if (rdev->num_crtc >= 6) {
5128 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5129 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5130 }
5131
5153550a
AD		 5132 if (rdev->num_crtc >= 2) {
5133 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5134 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5135 }
25a857fb
AD		 5136 if (rdev->num_crtc >= 4) {
5137 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5138 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5139 }
5140 if (rdev->num_crtc >= 6) {
5141 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5142 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5143 }
5144
5153550a
AD		 5145 if (!ASIC_IS_NODCE(rdev)) {
5146 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5147 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5148 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5149 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5150 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5151 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5152 }
25a857fb
AD		 5153
5154 return 0;
5155}
5156
5157static inline void si_irq_ack(struct radeon_device *rdev)
5158{
5159 u32 tmp;
5160
5153550a
AD		 5161 if (ASIC_IS_NODCE(rdev))
5162 return;
5163
25a857fb
AD		 5164 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5165 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5166 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5167 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5168 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5169 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5170 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5171 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5172 if (rdev->num_crtc >= 4) {
5173 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5174 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5175 }
5176 if (rdev->num_crtc >= 6) {
5177 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5178 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5179 }
5180
5181 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5182 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5183 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5184 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5185 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5186 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5187 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5188 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5189 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5190 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5191 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5192 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5193
5194 if (rdev->num_crtc >= 4) {
5195 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5196 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5197 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5198 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5199 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5200 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5201 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5202 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5203 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5204 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5205 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5206 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5207 }
5208
5209 if (rdev->num_crtc >= 6) {
5210 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5211 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5212 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5213 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5214 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5215 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5216 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5217 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5218 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5219 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5220 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5221 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5222 }
5223
5224 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5225 tmp = RREG32(DC_HPD1_INT_CONTROL);
5226 tmp |= DC_HPDx_INT_ACK;
5227 WREG32(DC_HPD1_INT_CONTROL, tmp);
5228 }
5229 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5230 tmp = RREG32(DC_HPD2_INT_CONTROL);
5231 tmp |= DC_HPDx_INT_ACK;
5232 WREG32(DC_HPD2_INT_CONTROL, tmp);
5233 }
5234 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5235 tmp = RREG32(DC_HPD3_INT_CONTROL);
5236 tmp |= DC_HPDx_INT_ACK;
5237 WREG32(DC_HPD3_INT_CONTROL, tmp);
5238 }
5239 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5240 tmp = RREG32(DC_HPD4_INT_CONTROL);
5241 tmp |= DC_HPDx_INT_ACK;
5242 WREG32(DC_HPD4_INT_CONTROL, tmp);
5243 }
5244 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5245 tmp = RREG32(DC_HPD5_INT_CONTROL);
5246 tmp |= DC_HPDx_INT_ACK;
5247 WREG32(DC_HPD5_INT_CONTROL, tmp);
5248 }
5249 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5250 tmp = RREG32(DC_HPD5_INT_CONTROL);
5251 tmp |= DC_HPDx_INT_ACK;
5252 WREG32(DC_HPD6_INT_CONTROL, tmp);
5253 }
5254}
5255
5256static void si_irq_disable(struct radeon_device *rdev)
5257{
5258 si_disable_interrupts(rdev);
5259 /* Wait and acknowledge irq */
5260 mdelay(1);
5261 si_irq_ack(rdev);
5262 si_disable_interrupt_state(rdev);
5263}
5264
5265static void si_irq_suspend(struct radeon_device *rdev)
5266{
5267 si_irq_disable(rdev);
5268 si_rlc_stop(rdev);
5269}
5270
9b136d51
AD		 5271static void si_irq_fini(struct radeon_device *rdev)
5272{
5273 si_irq_suspend(rdev);
5274 r600_ih_ring_fini(rdev);
5275}
5276
25a857fb
AD		 5277static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
5278{
5279 u32 wptr, tmp;
5280
5281 if (rdev->wb.enabled)
5282 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5283 else
5284 wptr = RREG32(IH_RB_WPTR);
5285
5286 if (wptr & RB_OVERFLOW) {
5287 /* When a ring buffer overflow happen start parsing interrupt
5288 * from the last not overwritten vector (wptr + 16). Hopefully
5289 * this should allow us to catchup.
5290 */
5291 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5292 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
5293 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5294 tmp = RREG32(IH_RB_CNTL);
5295 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5296 WREG32(IH_RB_CNTL, tmp);
5297 }
5298 return (wptr & rdev->ih.ptr_mask);
5299}
5300
5301/* SI IV Ring
5302 * Each IV ring entry is 128 bits:
5303 * [7:0] - interrupt source id
5304 * [31:8] - reserved
5305 * [59:32] - interrupt source data
5306 * [63:60] - reserved
5307 * [71:64] - RINGID
5308 * [79:72] - VMID
5309 * [127:80] - reserved
5310 */
5311int si_irq_process(struct radeon_device *rdev)
5312{
5313 u32 wptr;
5314 u32 rptr;
5315 u32 src_id, src_data, ring_id;
5316 u32 ring_index;
25a857fb
AD		 5317 bool queue_hotplug = false;
5318
5319 if (!rdev->ih.enabled || rdev->shutdown)
5320 return IRQ_NONE;
5321
5322 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 5323
5324restart_ih:
5325 /* is somebody else already processing irqs? */
5326 if (atomic_xchg(&rdev->ih.lock, 1))
5327 return IRQ_NONE;
5328
25a857fb
AD		 5329 rptr = rdev->ih.rptr;
5330 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
5331
25a857fb
AD		 5332 /* Order reading of wptr vs. reading of IH ring data */
5333 rmb();
5334
5335 /* display interrupts */
5336 si_irq_ack(rdev);
5337
25a857fb
AD		 5338 while (rptr != wptr) {
5339 /* wptr/rptr are in bytes! */
5340 ring_index = rptr / 4;
5341 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
5342 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
5343 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
5344
5345 switch (src_id) {
5346 case 1: /* D1 vblank/vline */
5347 switch (src_data) {
5348 case 0: /* D1 vblank */
5349 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
5350 if (rdev->irq.crtc_vblank_int[0]) {
5351 drm_handle_vblank(rdev->ddev, 0);
5352 rdev->pm.vblank_sync = true;
5353 wake_up(&rdev->irq.vblank_queue);
5354 }
736fc37f 5355 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 5356 radeon_crtc_handle_flip(rdev, 0);
5357 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
5358 DRM_DEBUG("IH: D1 vblank\n");
5359 }
5360 break;
5361 case 1: /* D1 vline */
5362 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
5363 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
5364 DRM_DEBUG("IH: D1 vline\n");
5365 }
5366 break;
5367 default:
5368 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5369 break;
5370 }
5371 break;
5372 case 2: /* D2 vblank/vline */
5373 switch (src_data) {
5374 case 0: /* D2 vblank */
5375 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
5376 if (rdev->irq.crtc_vblank_int[1]) {
5377 drm_handle_vblank(rdev->ddev, 1);
5378 rdev->pm.vblank_sync = true;
5379 wake_up(&rdev->irq.vblank_queue);
5380 }
736fc37f 5381 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 5382 radeon_crtc_handle_flip(rdev, 1);
5383 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5384 DRM_DEBUG("IH: D2 vblank\n");
5385 }
5386 break;
5387 case 1: /* D2 vline */
5388 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
5389 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5390 DRM_DEBUG("IH: D2 vline\n");
5391 }
5392 break;
5393 default:
5394 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5395 break;
5396 }
5397 break;
5398 case 3: /* D3 vblank/vline */
5399 switch (src_data) {
5400 case 0: /* D3 vblank */
5401 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
5402 if (rdev->irq.crtc_vblank_int[2]) {
5403 drm_handle_vblank(rdev->ddev, 2);
5404 rdev->pm.vblank_sync = true;
5405 wake_up(&rdev->irq.vblank_queue);
5406 }
736fc37f 5407 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 5408 radeon_crtc_handle_flip(rdev, 2);
5409 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5410 DRM_DEBUG("IH: D3 vblank\n");
5411 }
5412 break;
5413 case 1: /* D3 vline */
5414 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
5415 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5416 DRM_DEBUG("IH: D3 vline\n");
5417 }
5418 break;
5419 default:
5420 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5421 break;
5422 }
5423 break;
5424 case 4: /* D4 vblank/vline */
5425 switch (src_data) {
5426 case 0: /* D4 vblank */
5427 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
5428 if (rdev->irq.crtc_vblank_int[3]) {
5429 drm_handle_vblank(rdev->ddev, 3);
5430 rdev->pm.vblank_sync = true;
5431 wake_up(&rdev->irq.vblank_queue);
5432 }
736fc37f 5433 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 5434 radeon_crtc_handle_flip(rdev, 3);
5435 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5436 DRM_DEBUG("IH: D4 vblank\n");
5437 }
5438 break;
5439 case 1: /* D4 vline */
5440 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
5441 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5442 DRM_DEBUG("IH: D4 vline\n");
5443 }
5444 break;
5445 default:
5446 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5447 break;
5448 }
5449 break;
5450 case 5: /* D5 vblank/vline */
5451 switch (src_data) {
5452 case 0: /* D5 vblank */
5453 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
5454 if (rdev->irq.crtc_vblank_int[4]) {
5455 drm_handle_vblank(rdev->ddev, 4);
5456 rdev->pm.vblank_sync = true;
5457 wake_up(&rdev->irq.vblank_queue);
5458 }
736fc37f 5459 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 5460 radeon_crtc_handle_flip(rdev, 4);
5461 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5462 DRM_DEBUG("IH: D5 vblank\n");
5463 }
5464 break;
5465 case 1: /* D5 vline */
5466 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
5467 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5468 DRM_DEBUG("IH: D5 vline\n");
5469 }
5470 break;
5471 default:
5472 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5473 break;
5474 }
5475 break;
5476 case 6: /* D6 vblank/vline */
5477 switch (src_data) {
5478 case 0: /* D6 vblank */
5479 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
5480 if (rdev->irq.crtc_vblank_int[5]) {
5481 drm_handle_vblank(rdev->ddev, 5);
5482 rdev->pm.vblank_sync = true;
5483 wake_up(&rdev->irq.vblank_queue);
5484 }
736fc37f 5485 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 5486 radeon_crtc_handle_flip(rdev, 5);
5487 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5488 DRM_DEBUG("IH: D6 vblank\n");
5489 }
5490 break;
5491 case 1: /* D6 vline */
5492 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
5493 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5494 DRM_DEBUG("IH: D6 vline\n");
5495 }
5496 break;
5497 default:
5498 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5499 break;
5500 }
5501 break;
5502 case 42: /* HPD hotplug */
5503 switch (src_data) {
5504 case 0:
5505 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5506 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5507 queue_hotplug = true;
5508 DRM_DEBUG("IH: HPD1\n");
5509 }
5510 break;
5511 case 1:
5512 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5513 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5514 queue_hotplug = true;
5515 DRM_DEBUG("IH: HPD2\n");
5516 }
5517 break;
5518 case 2:
5519 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5520 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5521 queue_hotplug = true;
5522 DRM_DEBUG("IH: HPD3\n");
5523 }
5524 break;
5525 case 3:
5526 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5527 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5528 queue_hotplug = true;
5529 DRM_DEBUG("IH: HPD4\n");
5530 }
5531 break;
5532 case 4:
5533 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5534 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5535 queue_hotplug = true;
5536 DRM_DEBUG("IH: HPD5\n");
5537 }
5538 break;
5539 case 5:
5540 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5541 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5542 queue_hotplug = true;
5543 DRM_DEBUG("IH: HPD6\n");
5544 }
5545 break;
5546 default:
5547 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5548 break;
5549 }
5550 break;
ae133a11
CK		 5551 case 146:
5552 case 147:
5553 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5554 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5555 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5556 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5557 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5558 /* reset addr and status */
5559 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5560 break;
25a857fb
AD		 5561 case 176: /* RINGID0 CP_INT */
5562 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5563 break;
5564 case 177: /* RINGID1 CP_INT */
5565 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5566 break;
5567 case 178: /* RINGID2 CP_INT */
5568 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5569 break;
5570 case 181: /* CP EOP event */
5571 DRM_DEBUG("IH: CP EOP\n");
5572 switch (ring_id) {
5573 case 0:
5574 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5575 break;
5576 case 1:
5577 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5578 break;
5579 case 2:
5580 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5581 break;
5582 }
5583 break;
8c5fd7ef
AD		 5584 case 224: /* DMA trap event */
5585 DRM_DEBUG("IH: DMA trap\n");
5586 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5587 break;
25a857fb
AD		 5588 case 233: /* GUI IDLE */
5589 DRM_DEBUG("IH: GUI idle\n");
25a857fb 5590 break;
8c5fd7ef
AD		 5591 case 244: /* DMA trap event */
5592 DRM_DEBUG("IH: DMA1 trap\n");
5593 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5594 break;
25a857fb
AD		 5595 default:
5596 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5597 break;
5598 }
5599
5600 /* wptr/rptr are in bytes! */
5601 rptr += 16;
5602 rptr &= rdev->ih.ptr_mask;
5603 }
25a857fb
AD		 5604 if (queue_hotplug)
5605 schedule_work(&rdev->hotplug_work);
5606 rdev->ih.rptr = rptr;
5607 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 5608 atomic_set(&rdev->ih.lock, 0);
5609
5610 /* make sure wptr hasn't changed while processing */
5611 wptr = si_get_ih_wptr(rdev);
5612 if (wptr != rptr)
5613 goto restart_ih;
5614
25a857fb
AD		 5615 return IRQ_HANDLED;
5616}
5617
8c5fd7ef
AD		 5618/**
5619 * si_copy_dma - copy pages using the DMA engine
5620 *
5621 * @rdev: radeon_device pointer
5622 * @src_offset: src GPU address
5623 * @dst_offset: dst GPU address
5624 * @num_gpu_pages: number of GPU pages to xfer
5625 * @fence: radeon fence object
5626 *
5627 * Copy GPU paging using the DMA engine (SI).
5628 * Used by the radeon ttm implementation to move pages if
5629 * registered as the asic copy callback.
5630 */
5631int si_copy_dma(struct radeon_device *rdev,
5632 uint64_t src_offset, uint64_t dst_offset,
5633 unsigned num_gpu_pages,
5634 struct radeon_fence **fence)
5635{
5636 struct radeon_semaphore *sem = NULL;
5637 int ring_index = rdev->asic->copy.dma_ring_index;
5638 struct radeon_ring *ring = &rdev->ring[ring_index];
5639 u32 size_in_bytes, cur_size_in_bytes;
5640 int i, num_loops;
5641 int r = 0;
5642
5643 r = radeon_semaphore_create(rdev, &sem);
5644 if (r) {
5645 DRM_ERROR("radeon: moving bo (%d).\n", r);
5646 return r;
5647 }
5648
5649 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
5650 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
5651 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5652 if (r) {
5653 DRM_ERROR("radeon: moving bo (%d).\n", r);
5654 radeon_semaphore_free(rdev, &sem, NULL);
5655 return r;
5656 }
5657
5658 if (radeon_fence_need_sync(*fence, ring->idx)) {
5659 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5660 ring->idx);
5661 radeon_fence_note_sync(*fence, ring->idx);
5662 } else {
5663 radeon_semaphore_free(rdev, &sem, NULL);
5664 }
5665
5666 for (i = 0; i < num_loops; i++) {
5667 cur_size_in_bytes = size_in_bytes;
5668 if (cur_size_in_bytes > 0xFFFFF)
5669 cur_size_in_bytes = 0xFFFFF;
5670 size_in_bytes -= cur_size_in_bytes;
5671 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
5672 radeon_ring_write(ring, dst_offset & 0xffffffff);
5673 radeon_ring_write(ring, src_offset & 0xffffffff);
5674 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5675 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5676 src_offset += cur_size_in_bytes;
5677 dst_offset += cur_size_in_bytes;
5678 }
5679
5680 r = radeon_fence_emit(rdev, fence, ring->idx);
5681 if (r) {
5682 radeon_ring_unlock_undo(rdev, ring);
5683 return r;
5684 }
5685
5686 radeon_ring_unlock_commit(rdev, ring);
5687 radeon_semaphore_free(rdev, &sem, *fence);
5688
5689 return r;
5690}
5691
9b136d51
AD		 5692/*
5693 * startup/shutdown callbacks
5694 */
5695static int si_startup(struct radeon_device *rdev)
5696{
5697 struct radeon_ring *ring;
5698 int r;
5699
b9d305df
AD		 5700 /* enable pcie gen2/3 link */
5701 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 5702 /* enable aspm */
5703 si_program_aspm(rdev);
b9d305df 5704
9b136d51
AD		 5705 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5706 !rdev->rlc_fw || !rdev->mc_fw) {
5707 r = si_init_microcode(rdev);
5708 if (r) {
5709 DRM_ERROR("Failed to load firmware!\n");
5710 return r;
5711 }
5712 }
5713
5714 r = si_mc_load_microcode(rdev);
5715 if (r) {
5716 DRM_ERROR("Failed to load MC firmware!\n");
5717 return r;
5718 }
5719
5720 r = r600_vram_scratch_init(rdev);
5721 if (r)
5722 return r;
5723
5724 si_mc_program(rdev);
5725 r = si_pcie_gart_enable(rdev);
5726 if (r)
5727 return r;
5728 si_gpu_init(rdev);
5729
9b136d51
AD		 5730 /* allocate rlc buffers */
5731 r = si_rlc_init(rdev);
5732 if (r) {
5733 DRM_ERROR("Failed to init rlc BOs!\n");
5734 return r;
5735 }
5736
5737 /* allocate wb buffer */
5738 r = radeon_wb_init(rdev);
5739 if (r)
5740 return r;
5741
5742 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5743 if (r) {
5744 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5745 return r;
5746 }
5747
5748 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5749 if (r) {
5750 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5751 return r;
5752 }
5753
5754 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5755 if (r) {
5756 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5757 return r;
5758 }
5759
8c5fd7ef
AD		 5760 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5761 if (r) {
5762 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5763 return r;
5764 }
5765
5766 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5767 if (r) {
5768 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5769 return r;
5770 }
5771
1df0d523
AD		 5772 if (rdev->has_uvd) {
5773 r = rv770_uvd_resume(rdev);
5774 if (!r) {
5775 r = radeon_fence_driver_start_ring(rdev,
5776 R600_RING_TYPE_UVD_INDEX);
5777 if (r)
5778 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5779 }
f2ba57b5 5780 if (r)
1df0d523 5781 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 5782 }
f2ba57b5 5783
9b136d51 5784 /* Enable IRQ */
e49f3959
AH		 5785 if (!rdev->irq.installed) {
5786 r = radeon_irq_kms_init(rdev);
5787 if (r)
5788 return r;
5789 }
5790
9b136d51
AD		 5791 r = si_irq_init(rdev);
5792 if (r) {
5793 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5794 radeon_irq_kms_fini(rdev);
5795 return r;
5796 }
5797 si_irq_set(rdev);
5798
5799 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5800 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5801 CP_RB0_RPTR, CP_RB0_WPTR,
5802 0, 0xfffff, RADEON_CP_PACKET2);
5803 if (r)
5804 return r;
5805
5806 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5807 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
5808 CP_RB1_RPTR, CP_RB1_WPTR,
5809 0, 0xfffff, RADEON_CP_PACKET2);
5810 if (r)
5811 return r;
5812
5813 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5814 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
5815 CP_RB2_RPTR, CP_RB2_WPTR,
5816 0, 0xfffff, RADEON_CP_PACKET2);
5817 if (r)
5818 return r;
5819
8c5fd7ef
AD		 5820 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5821 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5822 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
5823 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
5824 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5825 if (r)
5826 return r;
5827
5828 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5829 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
5830 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
5831 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
5832 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5833 if (r)
5834 return r;
5835
9b136d51
AD		 5836 r = si_cp_load_microcode(rdev);
5837 if (r)
5838 return r;
5839 r = si_cp_resume(rdev);
5840 if (r)
5841 return r;
5842
8c5fd7ef
AD		 5843 r = cayman_dma_resume(rdev);
5844 if (r)
5845 return r;
5846
1df0d523
AD		 5847 if (rdev->has_uvd) {
5848 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5849 if (ring->ring_size) {
5850 r = radeon_ring_init(rdev, ring, ring->ring_size,
5851 R600_WB_UVD_RPTR_OFFSET,
5852 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5853 0, 0xfffff, RADEON_CP_PACKET2);
5854 if (!r)
5855 r = r600_uvd_init(rdev);
5856 if (r)
5857 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
5858 }
f2ba57b5
CK		 5859 }
5860
2898c348
CK		 5861 r = radeon_ib_pool_init(rdev);
5862 if (r) {
5863 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 5864 return r;
2898c348 5865 }
9b136d51 5866
c6105f24
CK		 5867 r = radeon_vm_manager_init(rdev);
5868 if (r) {
5869 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 5870 return r;
c6105f24 5871 }
9b136d51
AD		 5872
5873 return 0;
5874}
5875
5876int si_resume(struct radeon_device *rdev)
5877{
5878 int r;
5879
5880 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5881 * posting will perform necessary task to bring back GPU into good
5882 * shape.
5883 */
5884 /* post card */
5885 atom_asic_init(rdev->mode_info.atom_context);
5886
205996c0
AD		 5887 /* init golden registers */
5888 si_init_golden_registers(rdev);
5889
9b136d51
AD		 5890 rdev->accel_working = true;
5891 r = si_startup(rdev);
5892 if (r) {
5893 DRM_ERROR("si startup failed on resume\n");
5894 rdev->accel_working = false;
5895 return r;
5896 }
5897
5898 return r;
5899
5900}
5901
5902int si_suspend(struct radeon_device *rdev)
5903{
fa3daf9a 5904 radeon_vm_manager_fini(rdev);
9b136d51 5905 si_cp_enable(rdev, false);
8c5fd7ef 5906 cayman_dma_stop(rdev);
1df0d523
AD		 5907 if (rdev->has_uvd) {
5908 r600_uvd_rbc_stop(rdev);
5909 radeon_uvd_suspend(rdev);
5910 }
9b136d51
AD		 5911 si_irq_suspend(rdev);
5912 radeon_wb_disable(rdev);
5913 si_pcie_gart_disable(rdev);
5914 return 0;
5915}
5916
5917/* Plan is to move initialization in that function and use
5918 * helper function so that radeon_device_init pretty much
5919 * do nothing more than calling asic specific function. This
5920 * should also allow to remove a bunch of callback function
5921 * like vram_info.
5922 */
5923int si_init(struct radeon_device *rdev)
5924{
5925 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5926 int r;
5927
9b136d51
AD		 5928 /* Read BIOS */
5929 if (!radeon_get_bios(rdev)) {
5930 if (ASIC_IS_AVIVO(rdev))
5931 return -EINVAL;
5932 }
5933 /* Must be an ATOMBIOS */
5934 if (!rdev->is_atom_bios) {
5935 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
5936 return -EINVAL;
5937 }
5938 r = radeon_atombios_init(rdev);
5939 if (r)
5940 return r;
5941
5942 /* Post card if necessary */
5943 if (!radeon_card_posted(rdev)) {
5944 if (!rdev->bios) {
5945 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5946 return -EINVAL;
5947 }
5948 DRM_INFO("GPU not posted. posting now...\n");
5949 atom_asic_init(rdev->mode_info.atom_context);
5950 }
205996c0
AD		 5951 /* init golden registers */
5952 si_init_golden_registers(rdev);
9b136d51
AD		 5953 /* Initialize scratch registers */
5954 si_scratch_init(rdev);
5955 /* Initialize surface registers */
5956 radeon_surface_init(rdev);
5957 /* Initialize clocks */
5958 radeon_get_clock_info(rdev->ddev);
5959
5960 /* Fence driver */
5961 r = radeon_fence_driver_init(rdev);
5962 if (r)
5963 return r;
5964
5965 /* initialize memory controller */
5966 r = si_mc_init(rdev);
5967 if (r)
5968 return r;
5969 /* Memory manager */
5970 r = radeon_bo_init(rdev);
5971 if (r)
5972 return r;
5973
9b136d51
AD		 5974 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5975 ring->ring_obj = NULL;
5976 r600_ring_init(rdev, ring, 1024 * 1024);
5977
5978 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5979 ring->ring_obj = NULL;
5980 r600_ring_init(rdev, ring, 1024 * 1024);
5981
5982 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5983 ring->ring_obj = NULL;
5984 r600_ring_init(rdev, ring, 1024 * 1024);
5985
8c5fd7ef
AD		 5986 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5987 ring->ring_obj = NULL;
5988 r600_ring_init(rdev, ring, 64 * 1024);
5989
5990 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5991 ring->ring_obj = NULL;
5992 r600_ring_init(rdev, ring, 64 * 1024);
5993
1df0d523
AD		 5994 if (rdev->has_uvd) {
5995 r = radeon_uvd_init(rdev);
5996 if (!r) {
5997 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5998 ring->ring_obj = NULL;
5999 r600_ring_init(rdev, ring, 4096);
6000 }
f2ba57b5
CK		 6001 }
6002
9b136d51
AD		 6003 rdev->ih.ring_obj = NULL;
6004 r600_ih_ring_init(rdev, 64 * 1024);
6005
6006 r = r600_pcie_gart_init(rdev);
6007 if (r)
6008 return r;
6009
9b136d51 6010 rdev->accel_working = true;
9b136d51
AD		 6011 r = si_startup(rdev);
6012 if (r) {
6013 dev_err(rdev->dev, "disabling GPU acceleration\n");
6014 si_cp_fini(rdev);
8c5fd7ef 6015 cayman_dma_fini(rdev);
9b136d51
AD		 6016 si_irq_fini(rdev);
6017 si_rlc_fini(rdev);
6018 radeon_wb_fini(rdev);
2898c348 6019 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6020 radeon_vm_manager_fini(rdev);
6021 radeon_irq_kms_fini(rdev);
6022 si_pcie_gart_fini(rdev);
6023 rdev->accel_working = false;
6024 }
6025
6026 /* Don't start up if the MC ucode is missing.
6027 * The default clocks and voltages before the MC ucode
6028 * is loaded are not suffient for advanced operations.
6029 */
6030 if (!rdev->mc_fw) {
6031 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6032 return -EINVAL;
6033 }
6034
6035 return 0;
6036}
6037
6038void si_fini(struct radeon_device *rdev)
6039{
9b136d51 6040 si_cp_fini(rdev);
8c5fd7ef 6041 cayman_dma_fini(rdev);
9b136d51
AD		 6042 si_irq_fini(rdev);
6043 si_rlc_fini(rdev);
6044 radeon_wb_fini(rdev);
6045 radeon_vm_manager_fini(rdev);
2898c348 6046 radeon_ib_pool_fini(rdev);
9b136d51 6047 radeon_irq_kms_fini(rdev);
1df0d523
AD		 6048 if (rdev->has_uvd)
6049 radeon_uvd_fini(rdev);
9b136d51
AD		 6050 si_pcie_gart_fini(rdev);
6051 r600_vram_scratch_fini(rdev);
6052 radeon_gem_fini(rdev);
9b136d51
AD		 6053 radeon_fence_driver_fini(rdev);
6054 radeon_bo_fini(rdev);
6055 radeon_atombios_fini(rdev);
6056 kfree(rdev->bios);
6057 rdev->bios = NULL;
6058}
6059
6759a0a7 6060/**
d0418894 6061 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6062 *
6063 * @rdev: radeon_device pointer
6064 *
6065 * Fetches a GPU clock counter snapshot (SI).
6066 * Returns the 64 bit clock counter snapshot.
6067 */
d0418894 6068uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6069{
6070 uint64_t clock;
6071
6072 mutex_lock(&rdev->gpu_clock_mutex);
6073 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6074 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6075 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6076 mutex_unlock(&rdev->gpu_clock_mutex);
6077 return clock;
6078}
2539eb02 6079
2539eb02
CK		 6080int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6081{
facd112d 6082 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6083 int r;
6084
4ed10835
CK		 6085 /* bypass vclk and dclk with bclk */
6086 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6087 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6088 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6089
6090 /* put PLL in bypass mode */
6091 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6092
6093 if (!vclk || !dclk) {
6094 /* keep the Bypass mode, put PLL to sleep */
6095 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6096 return 0;
6097 }
6098
facd112d
CK		 6099 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6100 16384, 0x03FFFFFF, 0, 128, 5,
6101 &fb_div, &vclk_div, &dclk_div);
6102 if (r)
6103 return r;
2539eb02
CK		 6104
6105 /* set RESET_ANTI_MUX to 0 */
6106 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6107
6108 /* set VCO_MODE to 1 */
6109 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6110
6111 /* toggle UPLL_SLEEP to 1 then back to 0 */
6112 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6113 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6114
6115 /* deassert UPLL_RESET */
6116 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6117
6118 mdelay(1);
6119
facd112d 6120 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6121 if (r)
6122 return r;
6123
6124 /* assert UPLL_RESET again */
6125 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6126
6127 /* disable spread spectrum. */
6128 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6129
6130 /* set feedback divider */
facd112d 6131 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 6132
6133 /* set ref divider to 0 */
6134 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6135
facd112d 6136 if (fb_div < 307200)
2539eb02
CK		 6137 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6138 else
6139 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6140
6141 /* set PDIV_A and PDIV_B */
6142 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 6143 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 6144 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6145
6146 /* give the PLL some time to settle */
6147 mdelay(15);
6148
6149 /* deassert PLL_RESET */
6150 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6151
6152 mdelay(15);
6153
6154 /* switch from bypass mode to normal mode */
6155 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6156
facd112d 6157 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6158 if (r)
6159 return r;
6160
6161 /* switch VCLK and DCLK selection */
6162 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6163 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6164 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6165
6166 mdelay(100);
6167
6168 return 0;
6169}
b9d305df
AD		 6170
6171static void si_pcie_gen3_enable(struct radeon_device *rdev)
6172{
6173 struct pci_dev *root = rdev->pdev->bus->self;
6174 int bridge_pos, gpu_pos;
6175 u32 speed_cntl, mask, current_data_rate;
6176 int ret, i;
6177 u16 tmp16;
6178
6179 if (radeon_pcie_gen2 == 0)
6180 return;
6181
6182 if (rdev->flags & RADEON_IS_IGP)
6183 return;
6184
6185 if (!(rdev->flags & RADEON_IS_PCIE))
6186 return;
6187
6188 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6189 if (ret != 0)
6190 return;
6191
6192 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6193 return;
6194
6195 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6196 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6197 LC_CURRENT_DATA_RATE_SHIFT;
6198 if (mask & DRM_PCIE_SPEED_80) {
6199 if (current_data_rate == 2) {
6200 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6201 return;
6202 }
6203 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6204 } else if (mask & DRM_PCIE_SPEED_50) {
6205 if (current_data_rate == 1) {
6206 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6207 return;
6208 }
6209 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6210 }
6211
6212 bridge_pos = pci_pcie_cap(root);
6213 if (!bridge_pos)
6214 return;
6215
6216 gpu_pos = pci_pcie_cap(rdev->pdev);
6217 if (!gpu_pos)
6218 return;
6219
6220 if (mask & DRM_PCIE_SPEED_80) {
6221 /* re-try equalization if gen3 is not already enabled */
6222 if (current_data_rate != 2) {
6223 u16 bridge_cfg, gpu_cfg;
6224 u16 bridge_cfg2, gpu_cfg2;
6225 u32 max_lw, current_lw, tmp;
6226
6227 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6228 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6229
6230 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6231 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6232
6233 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6234 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6235
6236 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6237 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6238 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6239
6240 if (current_lw < max_lw) {
6241 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6242 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6243 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6244 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6245 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6246 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6247 }
6248 }
6249
6250 for (i = 0; i < 10; i++) {
6251 /* check status */
6252 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6253 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6254 break;
6255
6256 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6257 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6258
6259 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6260 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6261
6262 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6263 tmp |= LC_SET_QUIESCE;
6264 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6265
6266 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6267 tmp |= LC_REDO_EQ;
6268 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6269
6270 mdelay(100);
6271
6272 /* linkctl */
6273 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6274 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6275 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6276 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6277
6278 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6279 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6280 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6281 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6282
6283 /* linkctl2 */
6284 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6285 tmp16 &= ~((1 << 4) | (7 << 9));
6286 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6287 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6288
6289 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6290 tmp16 &= ~((1 << 4) | (7 << 9));
6291 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6292 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6293
6294 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6295 tmp &= ~LC_SET_QUIESCE;
6296 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6297 }
6298 }
6299 }
6300
6301 /* set the link speed */
6302 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6303 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6304 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6305
6306 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6307 tmp16 &= ~0xf;
6308 if (mask & DRM_PCIE_SPEED_80)
6309 tmp16 |= 3; /* gen3 */
6310 else if (mask & DRM_PCIE_SPEED_50)
6311 tmp16 |= 2; /* gen2 */
6312 else
6313 tmp16 |= 1; /* gen1 */
6314 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6315
6316 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6317 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
6318 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6319
6320 for (i = 0; i < rdev->usec_timeout; i++) {
6321 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6322 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
6323 break;
6324 udelay(1);
6325 }
6326}
6327
e0bcf165
AD		 6328static void si_program_aspm(struct radeon_device *rdev)
6329{
6330 u32 data, orig;
6331 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
6332 bool disable_clkreq = false;
6333
6334 if (!(rdev->flags & RADEON_IS_PCIE))
6335 return;
6336
6337 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6338 data &= ~LC_XMIT_N_FTS_MASK;
6339 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
6340 if (orig != data)
6341 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
6342
6343 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
6344 data |= LC_GO_TO_RECOVERY;
6345 if (orig != data)
6346 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
6347
6348 orig = data = RREG32_PCIE(PCIE_P_CNTL);
6349 data |= P_IGNORE_EDB_ERR;
6350 if (orig != data)
6351 WREG32_PCIE(PCIE_P_CNTL, data);
6352
6353 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6354 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
6355 data |= LC_PMI_TO_L1_DIS;
6356 if (!disable_l0s)
6357 data |= LC_L0S_INACTIVITY(7);
6358
6359 if (!disable_l1) {
6360 data |= LC_L1_INACTIVITY(7);
6361 data &= ~LC_PMI_TO_L1_DIS;
6362 if (orig != data)
6363 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6364
6365 if (!disable_plloff_in_l1) {
6366 bool clk_req_support;
6367
6368 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6369 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6370 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6371 if (orig != data)
6372 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6373
6374 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6375 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6376 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6377 if (orig != data)
6378 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6379
6380 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6381 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6382 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6383 if (orig != data)
6384 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6385
6386 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6387 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6388 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6389 if (orig != data)
6390 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6391
6392 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
6393 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6394 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6395 if (orig != data)
6396 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6397
6398 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6399 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6400 if (orig != data)
6401 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6402
6403 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
6404 data &= ~PLL_RAMP_UP_TIME_2_MASK;
6405 if (orig != data)
6406 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
6407
6408 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
6409 data &= ~PLL_RAMP_UP_TIME_3_MASK;
6410 if (orig != data)
6411 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
6412
6413 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6414 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6415 if (orig != data)
6416 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6417
6418 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6419 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6420 if (orig != data)
6421 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6422
6423 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
6424 data &= ~PLL_RAMP_UP_TIME_2_MASK;
6425 if (orig != data)
6426 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
6427
6428 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
6429 data &= ~PLL_RAMP_UP_TIME_3_MASK;
6430 if (orig != data)
6431 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
6432 }
6433 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6434 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
6435 data |= LC_DYN_LANES_PWR_STATE(3);
6436 if (orig != data)
6437 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
6438
6439 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
6440 data &= ~LS2_EXIT_TIME_MASK;
6441 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
6442 data |= LS2_EXIT_TIME(5);
6443 if (orig != data)
6444 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
6445
6446 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
6447 data &= ~LS2_EXIT_TIME_MASK;
6448 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
6449 data |= LS2_EXIT_TIME(5);
6450 if (orig != data)
6451 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
6452
6453 if (!disable_clkreq) {
6454 struct pci_dev *root = rdev->pdev->bus->self;
6455 u32 lnkcap;
6456
6457 clk_req_support = false;
6458 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
6459 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
6460 clk_req_support = true;
6461 } else {
6462 clk_req_support = false;
6463 }
6464
6465 if (clk_req_support) {
6466 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
6467 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
6468 if (orig != data)
6469 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
6470
6471 orig = data = RREG32(THM_CLK_CNTL);
6472 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
6473 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
6474 if (orig != data)
6475 WREG32(THM_CLK_CNTL, data);
6476
6477 orig = data = RREG32(MISC_CLK_CNTL);
6478 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
6479 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
6480 if (orig != data)
6481 WREG32(MISC_CLK_CNTL, data);
6482
6483 orig = data = RREG32(CG_CLKPIN_CNTL);
6484 data &= ~BCLK_AS_XCLK;
6485 if (orig != data)
6486 WREG32(CG_CLKPIN_CNTL, data);
6487
6488 orig = data = RREG32(CG_CLKPIN_CNTL_2);
6489 data &= ~FORCE_BIF_REFCLK_EN;
6490 if (orig != data)
6491 WREG32(CG_CLKPIN_CNTL_2, data);
6492
6493 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
6494 data &= ~MPLL_CLKOUT_SEL_MASK;
6495 data |= MPLL_CLKOUT_SEL(4);
6496 if (orig != data)
6497 WREG32(MPLL_BYPASSCLK_SEL, data);
6498
6499 orig = data = RREG32(SPLL_CNTL_MODE);
6500 data &= ~SPLL_REFCLK_SEL_MASK;
6501 if (orig != data)
6502 WREG32(SPLL_CNTL_MODE, data);
6503 }
6504 }
6505 } else {
6506 if (orig != data)
6507 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6508 }
6509
6510 orig = data = RREG32_PCIE(PCIE_CNTL2);
6511 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
6512 if (orig != data)
6513 WREG32_PCIE(PCIE_CNTL2, data);
6514
6515 if (!disable_l0s) {
6516 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6517 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
6518 data = RREG32_PCIE(PCIE_LC_STATUS1);
6519 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
6520 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6521 data &= ~LC_L0S_INACTIVITY_MASK;
6522 if (orig != data)
6523 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6524 }
6525 }
6526 }
6527}
Linux 6.x block layer and io_uring tree(s)