2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
30 #include "radeon_asic.h"
31 #include <drm/radeon_drm.h>
34 #include "si_blit_shaders.h"
36 #define SI_PFP_UCODE_SIZE 2144
37 #define SI_PM4_UCODE_SIZE 2144
38 #define SI_CE_UCODE_SIZE 2144
39 #define SI_RLC_UCODE_SIZE 2048
40 #define SI_MC_UCODE_SIZE 7769
41 #define OLAND_MC_UCODE_SIZE 7863
43 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
44 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
45 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
46 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
47 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
50 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
51 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
52 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
53 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
54 MODULE_FIRMWARE("radeon/VERDE_me.bin");
55 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
56 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
57 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
58 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
59 MODULE_FIRMWARE("radeon/OLAND_me.bin");
60 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
61 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
62 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
63 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
64 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
65 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
66 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
67 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
69 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
70 extern void r600_ih_ring_fini(struct radeon_device *rdev);
71 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
72 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
73 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
74 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
75 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
76 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
78 static const u32 tahiti_golden_rlc_registers[] =
80 0xc424, 0xffffffff, 0x00601005,
81 0xc47c, 0xffffffff, 0x10104040,
82 0xc488, 0xffffffff, 0x0100000a,
83 0xc314, 0xffffffff, 0x00000800,
84 0xc30c, 0xffffffff, 0x800000f4,
85 0xf4a8, 0xffffffff, 0x00000000
88 static const u32 tahiti_golden_registers[] =
90 0x9a10, 0x00010000, 0x00018208,
91 0x9830, 0xffffffff, 0x00000000,
92 0x9834, 0xf00fffff, 0x00000400,
93 0x9838, 0x0002021c, 0x00020200,
94 0xc78, 0x00000080, 0x00000000,
95 0xd030, 0x000300c0, 0x00800040,
96 0xd830, 0x000300c0, 0x00800040,
97 0x5bb0, 0x000000f0, 0x00000070,
98 0x5bc0, 0x00200000, 0x50100000,
99 0x7030, 0x31000311, 0x00000011,
100 0x277c, 0x00000003, 0x000007ff,
101 0x240c, 0x000007ff, 0x00000000,
102 0x8a14, 0xf000001f, 0x00000007,
103 0x8b24, 0xffffffff, 0x00ffffff,
104 0x8b10, 0x0000ff0f, 0x00000000,
105 0x28a4c, 0x07ffffff, 0x4e000000,
106 0x28350, 0x3f3f3fff, 0x2a00126a,
107 0x30, 0x000000ff, 0x0040,
108 0x34, 0x00000040, 0x00004040,
109 0x9100, 0x07ffffff, 0x03000000,
110 0x8e88, 0x01ff1f3f, 0x00000000,
111 0x8e84, 0x01ff1f3f, 0x00000000,
112 0x9060, 0x0000007f, 0x00000020,
113 0x9508, 0x00010000, 0x00010000,
114 0xac14, 0x00000200, 0x000002fb,
115 0xac10, 0xffffffff, 0x0000543b,
116 0xac0c, 0xffffffff, 0xa9210876,
117 0x88d0, 0xffffffff, 0x000fff40,
118 0x88d4, 0x0000001f, 0x00000010,
119 0x1410, 0x20000000, 0x20fffed8,
120 0x15c0, 0x000c0fc0, 0x000c0400
123 static const u32 tahiti_golden_registers2[] =
125 0xc64, 0x00000001, 0x00000001
128 static const u32 pitcairn_golden_rlc_registers[] =
130 0xc424, 0xffffffff, 0x00601004,
131 0xc47c, 0xffffffff, 0x10102020,
132 0xc488, 0xffffffff, 0x01000020,
133 0xc314, 0xffffffff, 0x00000800,
134 0xc30c, 0xffffffff, 0x800000a4
137 static const u32 pitcairn_golden_registers[] =
139 0x9a10, 0x00010000, 0x00018208,
140 0x9830, 0xffffffff, 0x00000000,
141 0x9834, 0xf00fffff, 0x00000400,
142 0x9838, 0x0002021c, 0x00020200,
143 0xc78, 0x00000080, 0x00000000,
144 0xd030, 0x000300c0, 0x00800040,
145 0xd830, 0x000300c0, 0x00800040,
146 0x5bb0, 0x000000f0, 0x00000070,
147 0x5bc0, 0x00200000, 0x50100000,
148 0x7030, 0x31000311, 0x00000011,
149 0x2ae4, 0x00073ffe, 0x000022a2,
150 0x240c, 0x000007ff, 0x00000000,
151 0x8a14, 0xf000001f, 0x00000007,
152 0x8b24, 0xffffffff, 0x00ffffff,
153 0x8b10, 0x0000ff0f, 0x00000000,
154 0x28a4c, 0x07ffffff, 0x4e000000,
155 0x28350, 0x3f3f3fff, 0x2a00126a,
156 0x30, 0x000000ff, 0x0040,
157 0x34, 0x00000040, 0x00004040,
158 0x9100, 0x07ffffff, 0x03000000,
159 0x9060, 0x0000007f, 0x00000020,
160 0x9508, 0x00010000, 0x00010000,
161 0xac14, 0x000003ff, 0x000000f7,
162 0xac10, 0xffffffff, 0x00000000,
163 0xac0c, 0xffffffff, 0x32761054,
164 0x88d4, 0x0000001f, 0x00000010,
165 0x15c0, 0x000c0fc0, 0x000c0400
168 static const u32 verde_golden_rlc_registers[] =
170 0xc424, 0xffffffff, 0x033f1005,
171 0xc47c, 0xffffffff, 0x10808020,
172 0xc488, 0xffffffff, 0x00800008,
173 0xc314, 0xffffffff, 0x00001000,
174 0xc30c, 0xffffffff, 0x80010014
177 static const u32 verde_golden_registers[] =
179 0x9a10, 0x00010000, 0x00018208,
180 0x9830, 0xffffffff, 0x00000000,
181 0x9834, 0xf00fffff, 0x00000400,
182 0x9838, 0x0002021c, 0x00020200,
183 0xc78, 0x00000080, 0x00000000,
184 0xd030, 0x000300c0, 0x00800040,
185 0xd030, 0x000300c0, 0x00800040,
186 0xd830, 0x000300c0, 0x00800040,
187 0xd830, 0x000300c0, 0x00800040,
188 0x5bb0, 0x000000f0, 0x00000070,
189 0x5bc0, 0x00200000, 0x50100000,
190 0x7030, 0x31000311, 0x00000011,
191 0x2ae4, 0x00073ffe, 0x000022a2,
192 0x2ae4, 0x00073ffe, 0x000022a2,
193 0x2ae4, 0x00073ffe, 0x000022a2,
194 0x240c, 0x000007ff, 0x00000000,
195 0x240c, 0x000007ff, 0x00000000,
196 0x240c, 0x000007ff, 0x00000000,
197 0x8a14, 0xf000001f, 0x00000007,
198 0x8a14, 0xf000001f, 0x00000007,
199 0x8a14, 0xf000001f, 0x00000007,
200 0x8b24, 0xffffffff, 0x00ffffff,
201 0x8b10, 0x0000ff0f, 0x00000000,
202 0x28a4c, 0x07ffffff, 0x4e000000,
203 0x28350, 0x3f3f3fff, 0x0000124a,
204 0x28350, 0x3f3f3fff, 0x0000124a,
205 0x28350, 0x3f3f3fff, 0x0000124a,
206 0x30, 0x000000ff, 0x0040,
207 0x34, 0x00000040, 0x00004040,
208 0x9100, 0x07ffffff, 0x03000000,
209 0x9100, 0x07ffffff, 0x03000000,
210 0x8e88, 0x01ff1f3f, 0x00000000,
211 0x8e88, 0x01ff1f3f, 0x00000000,
212 0x8e88, 0x01ff1f3f, 0x00000000,
213 0x8e84, 0x01ff1f3f, 0x00000000,
214 0x8e84, 0x01ff1f3f, 0x00000000,
215 0x8e84, 0x01ff1f3f, 0x00000000,
216 0x9060, 0x0000007f, 0x00000020,
217 0x9508, 0x00010000, 0x00010000,
218 0xac14, 0x000003ff, 0x00000003,
219 0xac14, 0x000003ff, 0x00000003,
220 0xac14, 0x000003ff, 0x00000003,
221 0xac10, 0xffffffff, 0x00000000,
222 0xac10, 0xffffffff, 0x00000000,
223 0xac10, 0xffffffff, 0x00000000,
224 0xac0c, 0xffffffff, 0x00001032,
225 0xac0c, 0xffffffff, 0x00001032,
226 0xac0c, 0xffffffff, 0x00001032,
227 0x88d4, 0x0000001f, 0x00000010,
228 0x88d4, 0x0000001f, 0x00000010,
229 0x88d4, 0x0000001f, 0x00000010,
230 0x15c0, 0x000c0fc0, 0x000c0400
233 static const u32 oland_golden_rlc_registers[] =
235 0xc424, 0xffffffff, 0x00601005,
236 0xc47c, 0xffffffff, 0x10104040,
237 0xc488, 0xffffffff, 0x0100000a,
238 0xc314, 0xffffffff, 0x00000800,
239 0xc30c, 0xffffffff, 0x800000f4
242 static const u32 oland_golden_registers[] =
244 0x9a10, 0x00010000, 0x00018208,
245 0x9830, 0xffffffff, 0x00000000,
246 0x9834, 0xf00fffff, 0x00000400,
247 0x9838, 0x0002021c, 0x00020200,
248 0xc78, 0x00000080, 0x00000000,
249 0xd030, 0x000300c0, 0x00800040,
250 0xd830, 0x000300c0, 0x00800040,
251 0x5bb0, 0x000000f0, 0x00000070,
252 0x5bc0, 0x00200000, 0x50100000,
253 0x7030, 0x31000311, 0x00000011,
254 0x2ae4, 0x00073ffe, 0x000022a2,
255 0x240c, 0x000007ff, 0x00000000,
256 0x8a14, 0xf000001f, 0x00000007,
257 0x8b24, 0xffffffff, 0x00ffffff,
258 0x8b10, 0x0000ff0f, 0x00000000,
259 0x28a4c, 0x07ffffff, 0x4e000000,
260 0x28350, 0x3f3f3fff, 0x00000082,
261 0x30, 0x000000ff, 0x0040,
262 0x34, 0x00000040, 0x00004040,
263 0x9100, 0x07ffffff, 0x03000000,
264 0x9060, 0x0000007f, 0x00000020,
265 0x9508, 0x00010000, 0x00010000,
266 0xac14, 0x000003ff, 0x000000f3,
267 0xac10, 0xffffffff, 0x00000000,
268 0xac0c, 0xffffffff, 0x00003210,
269 0x88d4, 0x0000001f, 0x00000010,
270 0x15c0, 0x000c0fc0, 0x000c0400
273 static const u32 tahiti_mgcg_cgcg_init[] =
275 0xc400, 0xffffffff, 0xfffffffc,
276 0x802c, 0xffffffff, 0xe0000000,
277 0x9a60, 0xffffffff, 0x00000100,
278 0x92a4, 0xffffffff, 0x00000100,
279 0xc164, 0xffffffff, 0x00000100,
280 0x9774, 0xffffffff, 0x00000100,
281 0x8984, 0xffffffff, 0x06000100,
282 0x8a18, 0xffffffff, 0x00000100,
283 0x92a0, 0xffffffff, 0x00000100,
284 0xc380, 0xffffffff, 0x00000100,
285 0x8b28, 0xffffffff, 0x00000100,
286 0x9144, 0xffffffff, 0x00000100,
287 0x8d88, 0xffffffff, 0x00000100,
288 0x8d8c, 0xffffffff, 0x00000100,
289 0x9030, 0xffffffff, 0x00000100,
290 0x9034, 0xffffffff, 0x00000100,
291 0x9038, 0xffffffff, 0x00000100,
292 0x903c, 0xffffffff, 0x00000100,
293 0xad80, 0xffffffff, 0x00000100,
294 0xac54, 0xffffffff, 0x00000100,
295 0x897c, 0xffffffff, 0x06000100,
296 0x9868, 0xffffffff, 0x00000100,
297 0x9510, 0xffffffff, 0x00000100,
298 0xaf04, 0xffffffff, 0x00000100,
299 0xae04, 0xffffffff, 0x00000100,
300 0x949c, 0xffffffff, 0x00000100,
301 0x802c, 0xffffffff, 0xe0000000,
302 0x9160, 0xffffffff, 0x00010000,
303 0x9164, 0xffffffff, 0x00030002,
304 0x9168, 0xffffffff, 0x00040007,
305 0x916c, 0xffffffff, 0x00060005,
306 0x9170, 0xffffffff, 0x00090008,
307 0x9174, 0xffffffff, 0x00020001,
308 0x9178, 0xffffffff, 0x00040003,
309 0x917c, 0xffffffff, 0x00000007,
310 0x9180, 0xffffffff, 0x00060005,
311 0x9184, 0xffffffff, 0x00090008,
312 0x9188, 0xffffffff, 0x00030002,
313 0x918c, 0xffffffff, 0x00050004,
314 0x9190, 0xffffffff, 0x00000008,
315 0x9194, 0xffffffff, 0x00070006,
316 0x9198, 0xffffffff, 0x000a0009,
317 0x919c, 0xffffffff, 0x00040003,
318 0x91a0, 0xffffffff, 0x00060005,
319 0x91a4, 0xffffffff, 0x00000009,
320 0x91a8, 0xffffffff, 0x00080007,
321 0x91ac, 0xffffffff, 0x000b000a,
322 0x91b0, 0xffffffff, 0x00050004,
323 0x91b4, 0xffffffff, 0x00070006,
324 0x91b8, 0xffffffff, 0x0008000b,
325 0x91bc, 0xffffffff, 0x000a0009,
326 0x91c0, 0xffffffff, 0x000d000c,
327 0x91c4, 0xffffffff, 0x00060005,
328 0x91c8, 0xffffffff, 0x00080007,
329 0x91cc, 0xffffffff, 0x0000000b,
330 0x91d0, 0xffffffff, 0x000a0009,
331 0x91d4, 0xffffffff, 0x000d000c,
332 0x91d8, 0xffffffff, 0x00070006,
333 0x91dc, 0xffffffff, 0x00090008,
334 0x91e0, 0xffffffff, 0x0000000c,
335 0x91e4, 0xffffffff, 0x000b000a,
336 0x91e8, 0xffffffff, 0x000e000d,
337 0x91ec, 0xffffffff, 0x00080007,
338 0x91f0, 0xffffffff, 0x000a0009,
339 0x91f4, 0xffffffff, 0x0000000d,
340 0x91f8, 0xffffffff, 0x000c000b,
341 0x91fc, 0xffffffff, 0x000f000e,
342 0x9200, 0xffffffff, 0x00090008,
343 0x9204, 0xffffffff, 0x000b000a,
344 0x9208, 0xffffffff, 0x000c000f,
345 0x920c, 0xffffffff, 0x000e000d,
346 0x9210, 0xffffffff, 0x00110010,
347 0x9214, 0xffffffff, 0x000a0009,
348 0x9218, 0xffffffff, 0x000c000b,
349 0x921c, 0xffffffff, 0x0000000f,
350 0x9220, 0xffffffff, 0x000e000d,
351 0x9224, 0xffffffff, 0x00110010,
352 0x9228, 0xffffffff, 0x000b000a,
353 0x922c, 0xffffffff, 0x000d000c,
354 0x9230, 0xffffffff, 0x00000010,
355 0x9234, 0xffffffff, 0x000f000e,
356 0x9238, 0xffffffff, 0x00120011,
357 0x923c, 0xffffffff, 0x000c000b,
358 0x9240, 0xffffffff, 0x000e000d,
359 0x9244, 0xffffffff, 0x00000011,
360 0x9248, 0xffffffff, 0x0010000f,
361 0x924c, 0xffffffff, 0x00130012,
362 0x9250, 0xffffffff, 0x000d000c,
363 0x9254, 0xffffffff, 0x000f000e,
364 0x9258, 0xffffffff, 0x00100013,
365 0x925c, 0xffffffff, 0x00120011,
366 0x9260, 0xffffffff, 0x00150014,
367 0x9264, 0xffffffff, 0x000e000d,
368 0x9268, 0xffffffff, 0x0010000f,
369 0x926c, 0xffffffff, 0x00000013,
370 0x9270, 0xffffffff, 0x00120011,
371 0x9274, 0xffffffff, 0x00150014,
372 0x9278, 0xffffffff, 0x000f000e,
373 0x927c, 0xffffffff, 0x00110010,
374 0x9280, 0xffffffff, 0x00000014,
375 0x9284, 0xffffffff, 0x00130012,
376 0x9288, 0xffffffff, 0x00160015,
377 0x928c, 0xffffffff, 0x0010000f,
378 0x9290, 0xffffffff, 0x00120011,
379 0x9294, 0xffffffff, 0x00000015,
380 0x9298, 0xffffffff, 0x00140013,
381 0x929c, 0xffffffff, 0x00170016,
382 0x9150, 0xffffffff, 0x96940200,
383 0x8708, 0xffffffff, 0x00900100,
384 0xc478, 0xffffffff, 0x00000080,
385 0xc404, 0xffffffff, 0x0020003f,
386 0x30, 0xffffffff, 0x0000001c,
387 0x34, 0x000f0000, 0x000f0000,
388 0x160c, 0xffffffff, 0x00000100,
389 0x1024, 0xffffffff, 0x00000100,
390 0x102c, 0x00000101, 0x00000000,
391 0x20a8, 0xffffffff, 0x00000104,
392 0x264c, 0x000c0000, 0x000c0000,
393 0x2648, 0x000c0000, 0x000c0000,
394 0x55e4, 0xff000fff, 0x00000100,
395 0x55e8, 0x00000001, 0x00000001,
396 0x2f50, 0x00000001, 0x00000001,
397 0x30cc, 0xc0000fff, 0x00000104,
398 0xc1e4, 0x00000001, 0x00000001,
399 0xd0c0, 0xfffffff0, 0x00000100,
400 0xd8c0, 0xfffffff0, 0x00000100
403 static const u32 pitcairn_mgcg_cgcg_init[] =
405 0xc400, 0xffffffff, 0xfffffffc,
406 0x802c, 0xffffffff, 0xe0000000,
407 0x9a60, 0xffffffff, 0x00000100,
408 0x92a4, 0xffffffff, 0x00000100,
409 0xc164, 0xffffffff, 0x00000100,
410 0x9774, 0xffffffff, 0x00000100,
411 0x8984, 0xffffffff, 0x06000100,
412 0x8a18, 0xffffffff, 0x00000100,
413 0x92a0, 0xffffffff, 0x00000100,
414 0xc380, 0xffffffff, 0x00000100,
415 0x8b28, 0xffffffff, 0x00000100,
416 0x9144, 0xffffffff, 0x00000100,
417 0x8d88, 0xffffffff, 0x00000100,
418 0x8d8c, 0xffffffff, 0x00000100,
419 0x9030, 0xffffffff, 0x00000100,
420 0x9034, 0xffffffff, 0x00000100,
421 0x9038, 0xffffffff, 0x00000100,
422 0x903c, 0xffffffff, 0x00000100,
423 0xad80, 0xffffffff, 0x00000100,
424 0xac54, 0xffffffff, 0x00000100,
425 0x897c, 0xffffffff, 0x06000100,
426 0x9868, 0xffffffff, 0x00000100,
427 0x9510, 0xffffffff, 0x00000100,
428 0xaf04, 0xffffffff, 0x00000100,
429 0xae04, 0xffffffff, 0x00000100,
430 0x949c, 0xffffffff, 0x00000100,
431 0x802c, 0xffffffff, 0xe0000000,
432 0x9160, 0xffffffff, 0x00010000,
433 0x9164, 0xffffffff, 0x00030002,
434 0x9168, 0xffffffff, 0x00040007,
435 0x916c, 0xffffffff, 0x00060005,
436 0x9170, 0xffffffff, 0x00090008,
437 0x9174, 0xffffffff, 0x00020001,
438 0x9178, 0xffffffff, 0x00040003,
439 0x917c, 0xffffffff, 0x00000007,
440 0x9180, 0xffffffff, 0x00060005,
441 0x9184, 0xffffffff, 0x00090008,
442 0x9188, 0xffffffff, 0x00030002,
443 0x918c, 0xffffffff, 0x00050004,
444 0x9190, 0xffffffff, 0x00000008,
445 0x9194, 0xffffffff, 0x00070006,
446 0x9198, 0xffffffff, 0x000a0009,
447 0x919c, 0xffffffff, 0x00040003,
448 0x91a0, 0xffffffff, 0x00060005,
449 0x91a4, 0xffffffff, 0x00000009,
450 0x91a8, 0xffffffff, 0x00080007,
451 0x91ac, 0xffffffff, 0x000b000a,
452 0x91b0, 0xffffffff, 0x00050004,
453 0x91b4, 0xffffffff, 0x00070006,
454 0x91b8, 0xffffffff, 0x0008000b,
455 0x91bc, 0xffffffff, 0x000a0009,
456 0x91c0, 0xffffffff, 0x000d000c,
457 0x9200, 0xffffffff, 0x00090008,
458 0x9204, 0xffffffff, 0x000b000a,
459 0x9208, 0xffffffff, 0x000c000f,
460 0x920c, 0xffffffff, 0x000e000d,
461 0x9210, 0xffffffff, 0x00110010,
462 0x9214, 0xffffffff, 0x000a0009,
463 0x9218, 0xffffffff, 0x000c000b,
464 0x921c, 0xffffffff, 0x0000000f,
465 0x9220, 0xffffffff, 0x000e000d,
466 0x9224, 0xffffffff, 0x00110010,
467 0x9228, 0xffffffff, 0x000b000a,
468 0x922c, 0xffffffff, 0x000d000c,
469 0x9230, 0xffffffff, 0x00000010,
470 0x9234, 0xffffffff, 0x000f000e,
471 0x9238, 0xffffffff, 0x00120011,
472 0x923c, 0xffffffff, 0x000c000b,
473 0x9240, 0xffffffff, 0x000e000d,
474 0x9244, 0xffffffff, 0x00000011,
475 0x9248, 0xffffffff, 0x0010000f,
476 0x924c, 0xffffffff, 0x00130012,
477 0x9250, 0xffffffff, 0x000d000c,
478 0x9254, 0xffffffff, 0x000f000e,
479 0x9258, 0xffffffff, 0x00100013,
480 0x925c, 0xffffffff, 0x00120011,
481 0x9260, 0xffffffff, 0x00150014,
482 0x9150, 0xffffffff, 0x96940200,
483 0x8708, 0xffffffff, 0x00900100,
484 0xc478, 0xffffffff, 0x00000080,
485 0xc404, 0xffffffff, 0x0020003f,
486 0x30, 0xffffffff, 0x0000001c,
487 0x34, 0x000f0000, 0x000f0000,
488 0x160c, 0xffffffff, 0x00000100,
489 0x1024, 0xffffffff, 0x00000100,
490 0x102c, 0x00000101, 0x00000000,
491 0x20a8, 0xffffffff, 0x00000104,
492 0x55e4, 0xff000fff, 0x00000100,
493 0x55e8, 0x00000001, 0x00000001,
494 0x2f50, 0x00000001, 0x00000001,
495 0x30cc, 0xc0000fff, 0x00000104,
496 0xc1e4, 0x00000001, 0x00000001,
497 0xd0c0, 0xfffffff0, 0x00000100,
498 0xd8c0, 0xfffffff0, 0x00000100
501 static const u32 verde_mgcg_cgcg_init[] =
503 0xc400, 0xffffffff, 0xfffffffc,
504 0x802c, 0xffffffff, 0xe0000000,
505 0x9a60, 0xffffffff, 0x00000100,
506 0x92a4, 0xffffffff, 0x00000100,
507 0xc164, 0xffffffff, 0x00000100,
508 0x9774, 0xffffffff, 0x00000100,
509 0x8984, 0xffffffff, 0x06000100,
510 0x8a18, 0xffffffff, 0x00000100,
511 0x92a0, 0xffffffff, 0x00000100,
512 0xc380, 0xffffffff, 0x00000100,
513 0x8b28, 0xffffffff, 0x00000100,
514 0x9144, 0xffffffff, 0x00000100,
515 0x8d88, 0xffffffff, 0x00000100,
516 0x8d8c, 0xffffffff, 0x00000100,
517 0x9030, 0xffffffff, 0x00000100,
518 0x9034, 0xffffffff, 0x00000100,
519 0x9038, 0xffffffff, 0x00000100,
520 0x903c, 0xffffffff, 0x00000100,
521 0xad80, 0xffffffff, 0x00000100,
522 0xac54, 0xffffffff, 0x00000100,
523 0x897c, 0xffffffff, 0x06000100,
524 0x9868, 0xffffffff, 0x00000100,
525 0x9510, 0xffffffff, 0x00000100,
526 0xaf04, 0xffffffff, 0x00000100,
527 0xae04, 0xffffffff, 0x00000100,
528 0x949c, 0xffffffff, 0x00000100,
529 0x802c, 0xffffffff, 0xe0000000,
530 0x9160, 0xffffffff, 0x00010000,
531 0x9164, 0xffffffff, 0x00030002,
532 0x9168, 0xffffffff, 0x00040007,
533 0x916c, 0xffffffff, 0x00060005,
534 0x9170, 0xffffffff, 0x00090008,
535 0x9174, 0xffffffff, 0x00020001,
536 0x9178, 0xffffffff, 0x00040003,
537 0x917c, 0xffffffff, 0x00000007,
538 0x9180, 0xffffffff, 0x00060005,
539 0x9184, 0xffffffff, 0x00090008,
540 0x9188, 0xffffffff, 0x00030002,
541 0x918c, 0xffffffff, 0x00050004,
542 0x9190, 0xffffffff, 0x00000008,
543 0x9194, 0xffffffff, 0x00070006,
544 0x9198, 0xffffffff, 0x000a0009,
545 0x919c, 0xffffffff, 0x00040003,
546 0x91a0, 0xffffffff, 0x00060005,
547 0x91a4, 0xffffffff, 0x00000009,
548 0x91a8, 0xffffffff, 0x00080007,
549 0x91ac, 0xffffffff, 0x000b000a,
550 0x91b0, 0xffffffff, 0x00050004,
551 0x91b4, 0xffffffff, 0x00070006,
552 0x91b8, 0xffffffff, 0x0008000b,
553 0x91bc, 0xffffffff, 0x000a0009,
554 0x91c0, 0xffffffff, 0x000d000c,
555 0x9200, 0xffffffff, 0x00090008,
556 0x9204, 0xffffffff, 0x000b000a,
557 0x9208, 0xffffffff, 0x000c000f,
558 0x920c, 0xffffffff, 0x000e000d,
559 0x9210, 0xffffffff, 0x00110010,
560 0x9214, 0xffffffff, 0x000a0009,
561 0x9218, 0xffffffff, 0x000c000b,
562 0x921c, 0xffffffff, 0x0000000f,
563 0x9220, 0xffffffff, 0x000e000d,
564 0x9224, 0xffffffff, 0x00110010,
565 0x9228, 0xffffffff, 0x000b000a,
566 0x922c, 0xffffffff, 0x000d000c,
567 0x9230, 0xffffffff, 0x00000010,
568 0x9234, 0xffffffff, 0x000f000e,
569 0x9238, 0xffffffff, 0x00120011,
570 0x923c, 0xffffffff, 0x000c000b,
571 0x9240, 0xffffffff, 0x000e000d,
572 0x9244, 0xffffffff, 0x00000011,
573 0x9248, 0xffffffff, 0x0010000f,
574 0x924c, 0xffffffff, 0x00130012,
575 0x9250, 0xffffffff, 0x000d000c,
576 0x9254, 0xffffffff, 0x000f000e,
577 0x9258, 0xffffffff, 0x00100013,
578 0x925c, 0xffffffff, 0x00120011,
579 0x9260, 0xffffffff, 0x00150014,
580 0x9150, 0xffffffff, 0x96940200,
581 0x8708, 0xffffffff, 0x00900100,
582 0xc478, 0xffffffff, 0x00000080,
583 0xc404, 0xffffffff, 0x0020003f,
584 0x30, 0xffffffff, 0x0000001c,
585 0x34, 0x000f0000, 0x000f0000,
586 0x160c, 0xffffffff, 0x00000100,
587 0x1024, 0xffffffff, 0x00000100,
588 0x102c, 0x00000101, 0x00000000,
589 0x20a8, 0xffffffff, 0x00000104,
590 0x264c, 0x000c0000, 0x000c0000,
591 0x2648, 0x000c0000, 0x000c0000,
592 0x55e4, 0xff000fff, 0x00000100,
593 0x55e8, 0x00000001, 0x00000001,
594 0x2f50, 0x00000001, 0x00000001,
595 0x30cc, 0xc0000fff, 0x00000104,
596 0xc1e4, 0x00000001, 0x00000001,
597 0xd0c0, 0xfffffff0, 0x00000100,
598 0xd8c0, 0xfffffff0, 0x00000100
601 static const u32 oland_mgcg_cgcg_init[] =
603 0xc400, 0xffffffff, 0xfffffffc,
604 0x802c, 0xffffffff, 0xe0000000,
605 0x9a60, 0xffffffff, 0x00000100,
606 0x92a4, 0xffffffff, 0x00000100,
607 0xc164, 0xffffffff, 0x00000100,
608 0x9774, 0xffffffff, 0x00000100,
609 0x8984, 0xffffffff, 0x06000100,
610 0x8a18, 0xffffffff, 0x00000100,
611 0x92a0, 0xffffffff, 0x00000100,
612 0xc380, 0xffffffff, 0x00000100,
613 0x8b28, 0xffffffff, 0x00000100,
614 0x9144, 0xffffffff, 0x00000100,
615 0x8d88, 0xffffffff, 0x00000100,
616 0x8d8c, 0xffffffff, 0x00000100,
617 0x9030, 0xffffffff, 0x00000100,
618 0x9034, 0xffffffff, 0x00000100,
619 0x9038, 0xffffffff, 0x00000100,
620 0x903c, 0xffffffff, 0x00000100,
621 0xad80, 0xffffffff, 0x00000100,
622 0xac54, 0xffffffff, 0x00000100,
623 0x897c, 0xffffffff, 0x06000100,
624 0x9868, 0xffffffff, 0x00000100,
625 0x9510, 0xffffffff, 0x00000100,
626 0xaf04, 0xffffffff, 0x00000100,
627 0xae04, 0xffffffff, 0x00000100,
628 0x949c, 0xffffffff, 0x00000100,
629 0x802c, 0xffffffff, 0xe0000000,
630 0x9160, 0xffffffff, 0x00010000,
631 0x9164, 0xffffffff, 0x00030002,
632 0x9168, 0xffffffff, 0x00040007,
633 0x916c, 0xffffffff, 0x00060005,
634 0x9170, 0xffffffff, 0x00090008,
635 0x9174, 0xffffffff, 0x00020001,
636 0x9178, 0xffffffff, 0x00040003,
637 0x917c, 0xffffffff, 0x00000007,
638 0x9180, 0xffffffff, 0x00060005,
639 0x9184, 0xffffffff, 0x00090008,
640 0x9188, 0xffffffff, 0x00030002,
641 0x918c, 0xffffffff, 0x00050004,
642 0x9190, 0xffffffff, 0x00000008,
643 0x9194, 0xffffffff, 0x00070006,
644 0x9198, 0xffffffff, 0x000a0009,
645 0x919c, 0xffffffff, 0x00040003,
646 0x91a0, 0xffffffff, 0x00060005,
647 0x91a4, 0xffffffff, 0x00000009,
648 0x91a8, 0xffffffff, 0x00080007,
649 0x91ac, 0xffffffff, 0x000b000a,
650 0x91b0, 0xffffffff, 0x00050004,
651 0x91b4, 0xffffffff, 0x00070006,
652 0x91b8, 0xffffffff, 0x0008000b,
653 0x91bc, 0xffffffff, 0x000a0009,
654 0x91c0, 0xffffffff, 0x000d000c,
655 0x91c4, 0xffffffff, 0x00060005,
656 0x91c8, 0xffffffff, 0x00080007,
657 0x91cc, 0xffffffff, 0x0000000b,
658 0x91d0, 0xffffffff, 0x000a0009,
659 0x91d4, 0xffffffff, 0x000d000c,
660 0x9150, 0xffffffff, 0x96940200,
661 0x8708, 0xffffffff, 0x00900100,
662 0xc478, 0xffffffff, 0x00000080,
663 0xc404, 0xffffffff, 0x0020003f,
664 0x30, 0xffffffff, 0x0000001c,
665 0x34, 0x000f0000, 0x000f0000,
666 0x160c, 0xffffffff, 0x00000100,
667 0x1024, 0xffffffff, 0x00000100,
668 0x102c, 0x00000101, 0x00000000,
669 0x20a8, 0xffffffff, 0x00000104,
670 0x264c, 0x000c0000, 0x000c0000,
671 0x2648, 0x000c0000, 0x000c0000,
672 0x55e4, 0xff000fff, 0x00000100,
673 0x55e8, 0x00000001, 0x00000001,
674 0x2f50, 0x00000001, 0x00000001,
675 0x30cc, 0xc0000fff, 0x00000104,
676 0xc1e4, 0x00000001, 0x00000001,
677 0xd0c0, 0xfffffff0, 0x00000100,
678 0xd8c0, 0xfffffff0, 0x00000100
681 static u32 verde_pg_init[] =
683 0x353c, 0xffffffff, 0x40000,
684 0x3538, 0xffffffff, 0x200010ff,
685 0x353c, 0xffffffff, 0x0,
686 0x353c, 0xffffffff, 0x0,
687 0x353c, 0xffffffff, 0x0,
688 0x353c, 0xffffffff, 0x0,
689 0x353c, 0xffffffff, 0x0,
690 0x353c, 0xffffffff, 0x7007,
691 0x3538, 0xffffffff, 0x300010ff,
692 0x353c, 0xffffffff, 0x0,
693 0x353c, 0xffffffff, 0x0,
694 0x353c, 0xffffffff, 0x0,
695 0x353c, 0xffffffff, 0x0,
696 0x353c, 0xffffffff, 0x0,
697 0x353c, 0xffffffff, 0x400000,
698 0x3538, 0xffffffff, 0x100010ff,
699 0x353c, 0xffffffff, 0x0,
700 0x353c, 0xffffffff, 0x0,
701 0x353c, 0xffffffff, 0x0,
702 0x353c, 0xffffffff, 0x0,
703 0x353c, 0xffffffff, 0x0,
704 0x353c, 0xffffffff, 0x120200,
705 0x3538, 0xffffffff, 0x500010ff,
706 0x353c, 0xffffffff, 0x0,
707 0x353c, 0xffffffff, 0x0,
708 0x353c, 0xffffffff, 0x0,
709 0x353c, 0xffffffff, 0x0,
710 0x353c, 0xffffffff, 0x0,
711 0x353c, 0xffffffff, 0x1e1e16,
712 0x3538, 0xffffffff, 0x600010ff,
713 0x353c, 0xffffffff, 0x0,
714 0x353c, 0xffffffff, 0x0,
715 0x353c, 0xffffffff, 0x0,
716 0x353c, 0xffffffff, 0x0,
717 0x353c, 0xffffffff, 0x0,
718 0x353c, 0xffffffff, 0x171f1e,
719 0x3538, 0xffffffff, 0x700010ff,
720 0x353c, 0xffffffff, 0x0,
721 0x353c, 0xffffffff, 0x0,
722 0x353c, 0xffffffff, 0x0,
723 0x353c, 0xffffffff, 0x0,
724 0x353c, 0xffffffff, 0x0,
725 0x353c, 0xffffffff, 0x0,
726 0x3538, 0xffffffff, 0x9ff,
727 0x3500, 0xffffffff, 0x0,
728 0x3504, 0xffffffff, 0x10000800,
729 0x3504, 0xffffffff, 0xf,
730 0x3504, 0xffffffff, 0xf,
731 0x3500, 0xffffffff, 0x4,
732 0x3504, 0xffffffff, 0x1000051e,
733 0x3504, 0xffffffff, 0xffff,
734 0x3504, 0xffffffff, 0xffff,
735 0x3500, 0xffffffff, 0x8,
736 0x3504, 0xffffffff, 0x80500,
737 0x3500, 0xffffffff, 0x12,
738 0x3504, 0xffffffff, 0x9050c,
739 0x3500, 0xffffffff, 0x1d,
740 0x3504, 0xffffffff, 0xb052c,
741 0x3500, 0xffffffff, 0x2a,
742 0x3504, 0xffffffff, 0x1053e,
743 0x3500, 0xffffffff, 0x2d,
744 0x3504, 0xffffffff, 0x10546,
745 0x3500, 0xffffffff, 0x30,
746 0x3504, 0xffffffff, 0xa054e,
747 0x3500, 0xffffffff, 0x3c,
748 0x3504, 0xffffffff, 0x1055f,
749 0x3500, 0xffffffff, 0x3f,
750 0x3504, 0xffffffff, 0x10567,
751 0x3500, 0xffffffff, 0x42,
752 0x3504, 0xffffffff, 0x1056f,
753 0x3500, 0xffffffff, 0x45,
754 0x3504, 0xffffffff, 0x10572,
755 0x3500, 0xffffffff, 0x48,
756 0x3504, 0xffffffff, 0x20575,
757 0x3500, 0xffffffff, 0x4c,
758 0x3504, 0xffffffff, 0x190801,
759 0x3500, 0xffffffff, 0x67,
760 0x3504, 0xffffffff, 0x1082a,
761 0x3500, 0xffffffff, 0x6a,
762 0x3504, 0xffffffff, 0x1b082d,
763 0x3500, 0xffffffff, 0x87,
764 0x3504, 0xffffffff, 0x310851,
765 0x3500, 0xffffffff, 0xba,
766 0x3504, 0xffffffff, 0x891,
767 0x3500, 0xffffffff, 0xbc,
768 0x3504, 0xffffffff, 0x893,
769 0x3500, 0xffffffff, 0xbe,
770 0x3504, 0xffffffff, 0x20895,
771 0x3500, 0xffffffff, 0xc2,
772 0x3504, 0xffffffff, 0x20899,
773 0x3500, 0xffffffff, 0xc6,
774 0x3504, 0xffffffff, 0x2089d,
775 0x3500, 0xffffffff, 0xca,
776 0x3504, 0xffffffff, 0x8a1,
777 0x3500, 0xffffffff, 0xcc,
778 0x3504, 0xffffffff, 0x8a3,
779 0x3500, 0xffffffff, 0xce,
780 0x3504, 0xffffffff, 0x308a5,
781 0x3500, 0xffffffff, 0xd3,
782 0x3504, 0xffffffff, 0x6d08cd,
783 0x3500, 0xffffffff, 0x142,
784 0x3504, 0xffffffff, 0x2000095a,
785 0x3504, 0xffffffff, 0x1,
786 0x3500, 0xffffffff, 0x144,
787 0x3504, 0xffffffff, 0x301f095b,
788 0x3500, 0xffffffff, 0x165,
789 0x3504, 0xffffffff, 0xc094d,
790 0x3500, 0xffffffff, 0x173,
791 0x3504, 0xffffffff, 0xf096d,
792 0x3500, 0xffffffff, 0x184,
793 0x3504, 0xffffffff, 0x15097f,
794 0x3500, 0xffffffff, 0x19b,
795 0x3504, 0xffffffff, 0xc0998,
796 0x3500, 0xffffffff, 0x1a9,
797 0x3504, 0xffffffff, 0x409a7,
798 0x3500, 0xffffffff, 0x1af,
799 0x3504, 0xffffffff, 0xcdc,
800 0x3500, 0xffffffff, 0x1b1,
801 0x3504, 0xffffffff, 0x800,
802 0x3508, 0xffffffff, 0x6c9b2000,
803 0x3510, 0xfc00, 0x2000,
804 0x3544, 0xffffffff, 0xfc0,
805 0x28d4, 0x00000100, 0x100
808 static void si_init_golden_registers(struct radeon_device *rdev)
810 switch (rdev->family) {
812 radeon_program_register_sequence(rdev,
813 tahiti_golden_registers,
814 (const u32)ARRAY_SIZE(tahiti_golden_registers));
815 radeon_program_register_sequence(rdev,
816 tahiti_golden_rlc_registers,
817 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
818 radeon_program_register_sequence(rdev,
819 tahiti_mgcg_cgcg_init,
820 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
821 radeon_program_register_sequence(rdev,
822 tahiti_golden_registers2,
823 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
826 radeon_program_register_sequence(rdev,
827 pitcairn_golden_registers,
828 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
829 radeon_program_register_sequence(rdev,
830 pitcairn_golden_rlc_registers,
831 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
832 radeon_program_register_sequence(rdev,
833 pitcairn_mgcg_cgcg_init,
834 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
837 radeon_program_register_sequence(rdev,
838 verde_golden_registers,
839 (const u32)ARRAY_SIZE(verde_golden_registers));
840 radeon_program_register_sequence(rdev,
841 verde_golden_rlc_registers,
842 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
843 radeon_program_register_sequence(rdev,
844 verde_mgcg_cgcg_init,
845 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
846 radeon_program_register_sequence(rdev,
848 (const u32)ARRAY_SIZE(verde_pg_init));
851 radeon_program_register_sequence(rdev,
852 oland_golden_registers,
853 (const u32)ARRAY_SIZE(oland_golden_registers));
854 radeon_program_register_sequence(rdev,
855 oland_golden_rlc_registers,
856 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
857 radeon_program_register_sequence(rdev,
858 oland_mgcg_cgcg_init,
859 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
866 #define PCIE_BUS_CLK 10000
867 #define TCLK (PCIE_BUS_CLK / 10)
870 * si_get_xclk - get the xclk
872 * @rdev: radeon_device pointer
874 * Returns the reference clock used by the gfx engine
877 u32 si_get_xclk(struct radeon_device *rdev)
879 u32 reference_clock = rdev->clock.spll.reference_freq;
882 tmp = RREG32(CG_CLKPIN_CNTL_2);
883 if (tmp & MUX_TCLK_TO_XCLK)
886 tmp = RREG32(CG_CLKPIN_CNTL);
887 if (tmp & XTALIN_DIVIDE)
888 return reference_clock / 4;
890 return reference_clock;
893 /* get temperature in millidegrees */
894 int si_get_temp(struct radeon_device *rdev)
899 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
905 actual_temp = temp & 0x1ff;
907 actual_temp = (actual_temp * 1000);
912 #define TAHITI_IO_MC_REGS_SIZE 36
914 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
915 {0x0000006f, 0x03044000},
916 {0x00000070, 0x0480c018},
917 {0x00000071, 0x00000040},
918 {0x00000072, 0x01000000},
919 {0x00000074, 0x000000ff},
920 {0x00000075, 0x00143400},
921 {0x00000076, 0x08ec0800},
922 {0x00000077, 0x040000cc},
923 {0x00000079, 0x00000000},
924 {0x0000007a, 0x21000409},
925 {0x0000007c, 0x00000000},
926 {0x0000007d, 0xe8000000},
927 {0x0000007e, 0x044408a8},
928 {0x0000007f, 0x00000003},
929 {0x00000080, 0x00000000},
930 {0x00000081, 0x01000000},
931 {0x00000082, 0x02000000},
932 {0x00000083, 0x00000000},
933 {0x00000084, 0xe3f3e4f4},
934 {0x00000085, 0x00052024},
935 {0x00000087, 0x00000000},
936 {0x00000088, 0x66036603},
937 {0x00000089, 0x01000000},
938 {0x0000008b, 0x1c0a0000},
939 {0x0000008c, 0xff010000},
940 {0x0000008e, 0xffffefff},
941 {0x0000008f, 0xfff3efff},
942 {0x00000090, 0xfff3efbf},
943 {0x00000094, 0x00101101},
944 {0x00000095, 0x00000fff},
945 {0x00000096, 0x00116fff},
946 {0x00000097, 0x60010000},
947 {0x00000098, 0x10010000},
948 {0x00000099, 0x00006000},
949 {0x0000009a, 0x00001000},
950 {0x0000009f, 0x00a77400}
953 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
954 {0x0000006f, 0x03044000},
955 {0x00000070, 0x0480c018},
956 {0x00000071, 0x00000040},
957 {0x00000072, 0x01000000},
958 {0x00000074, 0x000000ff},
959 {0x00000075, 0x00143400},
960 {0x00000076, 0x08ec0800},
961 {0x00000077, 0x040000cc},
962 {0x00000079, 0x00000000},
963 {0x0000007a, 0x21000409},
964 {0x0000007c, 0x00000000},
965 {0x0000007d, 0xe8000000},
966 {0x0000007e, 0x044408a8},
967 {0x0000007f, 0x00000003},
968 {0x00000080, 0x00000000},
969 {0x00000081, 0x01000000},
970 {0x00000082, 0x02000000},
971 {0x00000083, 0x00000000},
972 {0x00000084, 0xe3f3e4f4},
973 {0x00000085, 0x00052024},
974 {0x00000087, 0x00000000},
975 {0x00000088, 0x66036603},
976 {0x00000089, 0x01000000},
977 {0x0000008b, 0x1c0a0000},
978 {0x0000008c, 0xff010000},
979 {0x0000008e, 0xffffefff},
980 {0x0000008f, 0xfff3efff},
981 {0x00000090, 0xfff3efbf},
982 {0x00000094, 0x00101101},
983 {0x00000095, 0x00000fff},
984 {0x00000096, 0x00116fff},
985 {0x00000097, 0x60010000},
986 {0x00000098, 0x10010000},
987 {0x00000099, 0x00006000},
988 {0x0000009a, 0x00001000},
989 {0x0000009f, 0x00a47400}
992 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
993 {0x0000006f, 0x03044000},
994 {0x00000070, 0x0480c018},
995 {0x00000071, 0x00000040},
996 {0x00000072, 0x01000000},
997 {0x00000074, 0x000000ff},
998 {0x00000075, 0x00143400},
999 {0x00000076, 0x08ec0800},
1000 {0x00000077, 0x040000cc},
1001 {0x00000079, 0x00000000},
1002 {0x0000007a, 0x21000409},
1003 {0x0000007c, 0x00000000},
1004 {0x0000007d, 0xe8000000},
1005 {0x0000007e, 0x044408a8},
1006 {0x0000007f, 0x00000003},
1007 {0x00000080, 0x00000000},
1008 {0x00000081, 0x01000000},
1009 {0x00000082, 0x02000000},
1010 {0x00000083, 0x00000000},
1011 {0x00000084, 0xe3f3e4f4},
1012 {0x00000085, 0x00052024},
1013 {0x00000087, 0x00000000},
1014 {0x00000088, 0x66036603},
1015 {0x00000089, 0x01000000},
1016 {0x0000008b, 0x1c0a0000},
1017 {0x0000008c, 0xff010000},
1018 {0x0000008e, 0xffffefff},
1019 {0x0000008f, 0xfff3efff},
1020 {0x00000090, 0xfff3efbf},
1021 {0x00000094, 0x00101101},
1022 {0x00000095, 0x00000fff},
1023 {0x00000096, 0x00116fff},
1024 {0x00000097, 0x60010000},
1025 {0x00000098, 0x10010000},
1026 {0x00000099, 0x00006000},
1027 {0x0000009a, 0x00001000},
1028 {0x0000009f, 0x00a37400}
1031 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1032 {0x0000006f, 0x03044000},
1033 {0x00000070, 0x0480c018},
1034 {0x00000071, 0x00000040},
1035 {0x00000072, 0x01000000},
1036 {0x00000074, 0x000000ff},
1037 {0x00000075, 0x00143400},
1038 {0x00000076, 0x08ec0800},
1039 {0x00000077, 0x040000cc},
1040 {0x00000079, 0x00000000},
1041 {0x0000007a, 0x21000409},
1042 {0x0000007c, 0x00000000},
1043 {0x0000007d, 0xe8000000},
1044 {0x0000007e, 0x044408a8},
1045 {0x0000007f, 0x00000003},
1046 {0x00000080, 0x00000000},
1047 {0x00000081, 0x01000000},
1048 {0x00000082, 0x02000000},
1049 {0x00000083, 0x00000000},
1050 {0x00000084, 0xe3f3e4f4},
1051 {0x00000085, 0x00052024},
1052 {0x00000087, 0x00000000},
1053 {0x00000088, 0x66036603},
1054 {0x00000089, 0x01000000},
1055 {0x0000008b, 0x1c0a0000},
1056 {0x0000008c, 0xff010000},
1057 {0x0000008e, 0xffffefff},
1058 {0x0000008f, 0xfff3efff},
1059 {0x00000090, 0xfff3efbf},
1060 {0x00000094, 0x00101101},
1061 {0x00000095, 0x00000fff},
1062 {0x00000096, 0x00116fff},
1063 {0x00000097, 0x60010000},
1064 {0x00000098, 0x10010000},
1065 {0x00000099, 0x00006000},
1066 {0x0000009a, 0x00001000},
1067 {0x0000009f, 0x00a17730}
1070 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1071 {0x0000006f, 0x03044000},
1072 {0x00000070, 0x0480c018},
1073 {0x00000071, 0x00000040},
1074 {0x00000072, 0x01000000},
1075 {0x00000074, 0x000000ff},
1076 {0x00000075, 0x00143400},
1077 {0x00000076, 0x08ec0800},
1078 {0x00000077, 0x040000cc},
1079 {0x00000079, 0x00000000},
1080 {0x0000007a, 0x21000409},
1081 {0x0000007c, 0x00000000},
1082 {0x0000007d, 0xe8000000},
1083 {0x0000007e, 0x044408a8},
1084 {0x0000007f, 0x00000003},
1085 {0x00000080, 0x00000000},
1086 {0x00000081, 0x01000000},
1087 {0x00000082, 0x02000000},
1088 {0x00000083, 0x00000000},
1089 {0x00000084, 0xe3f3e4f4},
1090 {0x00000085, 0x00052024},
1091 {0x00000087, 0x00000000},
1092 {0x00000088, 0x66036603},
1093 {0x00000089, 0x01000000},
1094 {0x0000008b, 0x1c0a0000},
1095 {0x0000008c, 0xff010000},
1096 {0x0000008e, 0xffffefff},
1097 {0x0000008f, 0xfff3efff},
1098 {0x00000090, 0xfff3efbf},
1099 {0x00000094, 0x00101101},
1100 {0x00000095, 0x00000fff},
1101 {0x00000096, 0x00116fff},
1102 {0x00000097, 0x60010000},
1103 {0x00000098, 0x10010000},
1104 {0x00000099, 0x00006000},
1105 {0x0000009a, 0x00001000},
1106 {0x0000009f, 0x00a07730}
1110 static int si_mc_load_microcode(struct radeon_device *rdev)
1112 const __be32 *fw_data;
1113 u32 running, blackout = 0;
1115 int i, ucode_size, regs_size;
1120 switch (rdev->family) {
1122 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1123 ucode_size = SI_MC_UCODE_SIZE;
1124 regs_size = TAHITI_IO_MC_REGS_SIZE;
1127 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1128 ucode_size = SI_MC_UCODE_SIZE;
1129 regs_size = TAHITI_IO_MC_REGS_SIZE;
1133 io_mc_regs = (u32 *)&verde_io_mc_regs;
1134 ucode_size = SI_MC_UCODE_SIZE;
1135 regs_size = TAHITI_IO_MC_REGS_SIZE;
1138 io_mc_regs = (u32 *)&oland_io_mc_regs;
1139 ucode_size = OLAND_MC_UCODE_SIZE;
1140 regs_size = TAHITI_IO_MC_REGS_SIZE;
1143 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1144 ucode_size = OLAND_MC_UCODE_SIZE;
1145 regs_size = TAHITI_IO_MC_REGS_SIZE;
1149 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1153 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1154 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1157 /* reset the engine and set to writable */
1158 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1159 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1161 /* load mc io regs */
1162 for (i = 0; i < regs_size; i++) {
1163 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1164 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1166 /* load the MC ucode */
1167 fw_data = (const __be32 *)rdev->mc_fw->data;
1168 for (i = 0; i < ucode_size; i++)
1169 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1171 /* put the engine back into the active state */
1172 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1173 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1174 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1176 /* wait for training to complete */
1177 for (i = 0; i < rdev->usec_timeout; i++) {
1178 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1182 for (i = 0; i < rdev->usec_timeout; i++) {
1183 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1189 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1195 static int si_init_microcode(struct radeon_device *rdev)
1197 struct platform_device *pdev;
1198 const char *chip_name;
1199 const char *rlc_chip_name;
1200 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1206 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1209 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1213 switch (rdev->family) {
1215 chip_name = "TAHITI";
1216 rlc_chip_name = "TAHITI";
1217 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1218 me_req_size = SI_PM4_UCODE_SIZE * 4;
1219 ce_req_size = SI_CE_UCODE_SIZE * 4;
1220 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1221 mc_req_size = SI_MC_UCODE_SIZE * 4;
1224 chip_name = "PITCAIRN";
1225 rlc_chip_name = "PITCAIRN";
1226 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1227 me_req_size = SI_PM4_UCODE_SIZE * 4;
1228 ce_req_size = SI_CE_UCODE_SIZE * 4;
1229 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1230 mc_req_size = SI_MC_UCODE_SIZE * 4;
1233 chip_name = "VERDE";
1234 rlc_chip_name = "VERDE";
1235 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1236 me_req_size = SI_PM4_UCODE_SIZE * 4;
1237 ce_req_size = SI_CE_UCODE_SIZE * 4;
1238 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1239 mc_req_size = SI_MC_UCODE_SIZE * 4;
1242 chip_name = "OLAND";
1243 rlc_chip_name = "OLAND";
1244 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1245 me_req_size = SI_PM4_UCODE_SIZE * 4;
1246 ce_req_size = SI_CE_UCODE_SIZE * 4;
1247 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1248 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1251 chip_name = "HAINAN";
1252 rlc_chip_name = "HAINAN";
1253 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1254 me_req_size = SI_PM4_UCODE_SIZE * 4;
1255 ce_req_size = SI_CE_UCODE_SIZE * 4;
1256 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1257 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1262 DRM_INFO("Loading %s Microcode\n", chip_name);
1264 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1265 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1268 if (rdev->pfp_fw->size != pfp_req_size) {
1270 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1271 rdev->pfp_fw->size, fw_name);
1276 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1277 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1280 if (rdev->me_fw->size != me_req_size) {
1282 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1283 rdev->me_fw->size, fw_name);
1287 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1288 err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
1291 if (rdev->ce_fw->size != ce_req_size) {
1293 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1294 rdev->ce_fw->size, fw_name);
1298 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1299 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1302 if (rdev->rlc_fw->size != rlc_req_size) {
1304 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1305 rdev->rlc_fw->size, fw_name);
1309 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1310 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
1313 if (rdev->mc_fw->size != mc_req_size) {
1315 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1316 rdev->mc_fw->size, fw_name);
1321 platform_device_unregister(pdev);
1326 "si_cp: Failed to load firmware \"%s\"\n",
1328 release_firmware(rdev->pfp_fw);
1329 rdev->pfp_fw = NULL;
1330 release_firmware(rdev->me_fw);
1332 release_firmware(rdev->ce_fw);
1334 release_firmware(rdev->rlc_fw);
1335 rdev->rlc_fw = NULL;
1336 release_firmware(rdev->mc_fw);
1342 /* watermark setup */
1343 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1344 struct radeon_crtc *radeon_crtc,
1345 struct drm_display_mode *mode,
1346 struct drm_display_mode *other_mode)
1351 * There are 3 line buffers, each one shared by 2 display controllers.
1352 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1353 * the display controllers. The paritioning is done via one of four
1354 * preset allocations specified in bits 21:20:
1356 * 2 - whole lb, other crtc must be disabled
1358 /* this can get tricky if we have two large displays on a paired group
1359 * of crtcs. Ideally for multiple large displays we'd assign them to
1360 * non-linked crtcs for maximum line buffer allocation.
1362 if (radeon_crtc->base.enabled && mode) {
1366 tmp = 2; /* whole */
1370 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1371 DC_LB_MEMORY_CONFIG(tmp));
1373 if (radeon_crtc->base.enabled && mode) {
1383 /* controller not enabled, so no lb used */
1387 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1389 u32 tmp = RREG32(MC_SHARED_CHMAP);
1391 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1414 struct dce6_wm_params {
1415 u32 dram_channels; /* number of dram channels */
1416 u32 yclk; /* bandwidth per dram data pin in kHz */
1417 u32 sclk; /* engine clock in kHz */
1418 u32 disp_clk; /* display clock in kHz */
1419 u32 src_width; /* viewport width */
1420 u32 active_time; /* active display time in ns */
1421 u32 blank_time; /* blank time in ns */
1422 bool interlaced; /* mode is interlaced */
1423 fixed20_12 vsc; /* vertical scale ratio */
1424 u32 num_heads; /* number of active crtcs */
1425 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1426 u32 lb_size; /* line buffer allocated to pipe */
1427 u32 vtaps; /* vertical scaler taps */
1430 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1432 /* Calculate raw DRAM Bandwidth */
1433 fixed20_12 dram_efficiency; /* 0.7 */
1434 fixed20_12 yclk, dram_channels, bandwidth;
1437 a.full = dfixed_const(1000);
1438 yclk.full = dfixed_const(wm->yclk);
1439 yclk.full = dfixed_div(yclk, a);
1440 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1441 a.full = dfixed_const(10);
1442 dram_efficiency.full = dfixed_const(7);
1443 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1444 bandwidth.full = dfixed_mul(dram_channels, yclk);
1445 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1447 return dfixed_trunc(bandwidth);
1450 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1452 /* Calculate DRAM Bandwidth and the part allocated to display. */
1453 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1454 fixed20_12 yclk, dram_channels, bandwidth;
1457 a.full = dfixed_const(1000);
1458 yclk.full = dfixed_const(wm->yclk);
1459 yclk.full = dfixed_div(yclk, a);
1460 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1461 a.full = dfixed_const(10);
1462 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1463 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1464 bandwidth.full = dfixed_mul(dram_channels, yclk);
1465 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1467 return dfixed_trunc(bandwidth);
1470 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1472 /* Calculate the display Data return Bandwidth */
1473 fixed20_12 return_efficiency; /* 0.8 */
1474 fixed20_12 sclk, bandwidth;
1477 a.full = dfixed_const(1000);
1478 sclk.full = dfixed_const(wm->sclk);
1479 sclk.full = dfixed_div(sclk, a);
1480 a.full = dfixed_const(10);
1481 return_efficiency.full = dfixed_const(8);
1482 return_efficiency.full = dfixed_div(return_efficiency, a);
1483 a.full = dfixed_const(32);
1484 bandwidth.full = dfixed_mul(a, sclk);
1485 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1487 return dfixed_trunc(bandwidth);
1490 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1495 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1497 /* Calculate the DMIF Request Bandwidth */
1498 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1499 fixed20_12 disp_clk, sclk, bandwidth;
1500 fixed20_12 a, b1, b2;
1503 a.full = dfixed_const(1000);
1504 disp_clk.full = dfixed_const(wm->disp_clk);
1505 disp_clk.full = dfixed_div(disp_clk, a);
1506 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1507 b1.full = dfixed_mul(a, disp_clk);
1509 a.full = dfixed_const(1000);
1510 sclk.full = dfixed_const(wm->sclk);
1511 sclk.full = dfixed_div(sclk, a);
1512 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1513 b2.full = dfixed_mul(a, sclk);
1515 a.full = dfixed_const(10);
1516 disp_clk_request_efficiency.full = dfixed_const(8);
1517 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1519 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1521 a.full = dfixed_const(min_bandwidth);
1522 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1524 return dfixed_trunc(bandwidth);
1527 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1529 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1530 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1531 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1532 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1534 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1537 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1539 /* Calculate the display mode Average Bandwidth
1540 * DisplayMode should contain the source and destination dimensions,
1544 fixed20_12 line_time;
1545 fixed20_12 src_width;
1546 fixed20_12 bandwidth;
1549 a.full = dfixed_const(1000);
1550 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1551 line_time.full = dfixed_div(line_time, a);
1552 bpp.full = dfixed_const(wm->bytes_per_pixel);
1553 src_width.full = dfixed_const(wm->src_width);
1554 bandwidth.full = dfixed_mul(src_width, bpp);
1555 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1556 bandwidth.full = dfixed_div(bandwidth, line_time);
1558 return dfixed_trunc(bandwidth);
1561 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1563 /* First calcualte the latency in ns */
1564 u32 mc_latency = 2000; /* 2000 ns. */
1565 u32 available_bandwidth = dce6_available_bandwidth(wm);
1566 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1567 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1568 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1569 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1570 (wm->num_heads * cursor_line_pair_return_time);
1571 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1572 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1573 u32 tmp, dmif_size = 12288;
1576 if (wm->num_heads == 0)
1579 a.full = dfixed_const(2);
1580 b.full = dfixed_const(1);
1581 if ((wm->vsc.full > a.full) ||
1582 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1584 ((wm->vsc.full >= a.full) && wm->interlaced))
1585 max_src_lines_per_dst_line = 4;
1587 max_src_lines_per_dst_line = 2;
1589 a.full = dfixed_const(available_bandwidth);
1590 b.full = dfixed_const(wm->num_heads);
1591 a.full = dfixed_div(a, b);
1593 b.full = dfixed_const(mc_latency + 512);
1594 c.full = dfixed_const(wm->disp_clk);
1595 b.full = dfixed_div(b, c);
1597 c.full = dfixed_const(dmif_size);
1598 b.full = dfixed_div(c, b);
1600 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1602 b.full = dfixed_const(1000);
1603 c.full = dfixed_const(wm->disp_clk);
1604 b.full = dfixed_div(c, b);
1605 c.full = dfixed_const(wm->bytes_per_pixel);
1606 b.full = dfixed_mul(b, c);
1608 lb_fill_bw = min(tmp, dfixed_trunc(b));
1610 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1611 b.full = dfixed_const(1000);
1612 c.full = dfixed_const(lb_fill_bw);
1613 b.full = dfixed_div(c, b);
1614 a.full = dfixed_div(a, b);
1615 line_fill_time = dfixed_trunc(a);
1617 if (line_fill_time < wm->active_time)
1620 return latency + (line_fill_time - wm->active_time);
1624 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1626 if (dce6_average_bandwidth(wm) <=
1627 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1633 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
1635 if (dce6_average_bandwidth(wm) <=
1636 (dce6_available_bandwidth(wm) / wm->num_heads))
1642 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
1644 u32 lb_partitions = wm->lb_size / wm->src_width;
1645 u32 line_time = wm->active_time + wm->blank_time;
1646 u32 latency_tolerant_lines;
1650 a.full = dfixed_const(1);
1651 if (wm->vsc.full > a.full)
1652 latency_tolerant_lines = 1;
1654 if (lb_partitions <= (wm->vtaps + 1))
1655 latency_tolerant_lines = 1;
1657 latency_tolerant_lines = 2;
1660 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
1662 if (dce6_latency_watermark(wm) <= latency_hiding)
1668 static void dce6_program_watermarks(struct radeon_device *rdev,
1669 struct radeon_crtc *radeon_crtc,
1670 u32 lb_size, u32 num_heads)
1672 struct drm_display_mode *mode = &radeon_crtc->base.mode;
1673 struct dce6_wm_params wm;
1676 u32 latency_watermark_a = 0, latency_watermark_b = 0;
1677 u32 priority_a_mark = 0, priority_b_mark = 0;
1678 u32 priority_a_cnt = PRIORITY_OFF;
1679 u32 priority_b_cnt = PRIORITY_OFF;
1680 u32 tmp, arb_control3;
1683 if (radeon_crtc->base.enabled && num_heads && mode) {
1684 pixel_period = 1000000 / (u32)mode->clock;
1685 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
1689 wm.yclk = rdev->pm.current_mclk * 10;
1690 wm.sclk = rdev->pm.current_sclk * 10;
1691 wm.disp_clk = mode->clock;
1692 wm.src_width = mode->crtc_hdisplay;
1693 wm.active_time = mode->crtc_hdisplay * pixel_period;
1694 wm.blank_time = line_time - wm.active_time;
1695 wm.interlaced = false;
1696 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1697 wm.interlaced = true;
1698 wm.vsc = radeon_crtc->vsc;
1700 if (radeon_crtc->rmx_type != RMX_OFF)
1702 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
1703 wm.lb_size = lb_size;
1704 if (rdev->family == CHIP_ARUBA)
1705 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
1707 wm.dram_channels = si_get_number_of_dram_channels(rdev);
1708 wm.num_heads = num_heads;
1710 /* set for high clocks */
1711 latency_watermark_a = min(dce6_latency_watermark(&wm), (u32)65535);
1712 /* set for low clocks */
1713 /* wm.yclk = low clk; wm.sclk = low clk */
1714 latency_watermark_b = min(dce6_latency_watermark(&wm), (u32)65535);
1716 /* possibly force display priority to high */
1717 /* should really do this at mode validation time... */
1718 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
1719 !dce6_average_bandwidth_vs_available_bandwidth(&wm) ||
1720 !dce6_check_latency_hiding(&wm) ||
1721 (rdev->disp_priority == 2)) {
1722 DRM_DEBUG_KMS("force priority to high\n");
1723 priority_a_cnt |= PRIORITY_ALWAYS_ON;
1724 priority_b_cnt |= PRIORITY_ALWAYS_ON;
1727 a.full = dfixed_const(1000);
1728 b.full = dfixed_const(mode->clock);
1729 b.full = dfixed_div(b, a);
1730 c.full = dfixed_const(latency_watermark_a);
1731 c.full = dfixed_mul(c, b);
1732 c.full = dfixed_mul(c, radeon_crtc->hsc);
1733 c.full = dfixed_div(c, a);
1734 a.full = dfixed_const(16);
1735 c.full = dfixed_div(c, a);
1736 priority_a_mark = dfixed_trunc(c);
1737 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
1739 a.full = dfixed_const(1000);
1740 b.full = dfixed_const(mode->clock);
1741 b.full = dfixed_div(b, a);
1742 c.full = dfixed_const(latency_watermark_b);
1743 c.full = dfixed_mul(c, b);
1744 c.full = dfixed_mul(c, radeon_crtc->hsc);
1745 c.full = dfixed_div(c, a);
1746 a.full = dfixed_const(16);
1747 c.full = dfixed_div(c, a);
1748 priority_b_mark = dfixed_trunc(c);
1749 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
1753 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1755 tmp &= ~LATENCY_WATERMARK_MASK(3);
1756 tmp |= LATENCY_WATERMARK_MASK(1);
1757 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1758 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1759 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
1760 LATENCY_HIGH_WATERMARK(line_time)));
1762 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1763 tmp &= ~LATENCY_WATERMARK_MASK(3);
1764 tmp |= LATENCY_WATERMARK_MASK(2);
1765 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1766 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1767 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
1768 LATENCY_HIGH_WATERMARK(line_time)));
1769 /* restore original selection */
1770 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
1772 /* write the priority marks */
1773 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
1774 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
1778 void dce6_bandwidth_update(struct radeon_device *rdev)
1780 struct drm_display_mode *mode0 = NULL;
1781 struct drm_display_mode *mode1 = NULL;
1782 u32 num_heads = 0, lb_size;
1785 radeon_update_display_priority(rdev);
1787 for (i = 0; i < rdev->num_crtc; i++) {
1788 if (rdev->mode_info.crtcs[i]->base.enabled)
1791 for (i = 0; i < rdev->num_crtc; i += 2) {
1792 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
1793 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
1794 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
1795 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
1796 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
1797 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
1804 static void si_tiling_mode_table_init(struct radeon_device *rdev)
1806 const u32 num_tile_mode_states = 32;
1807 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1809 switch (rdev->config.si.mem_row_size_in_kb) {
1811 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1815 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1818 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1822 if ((rdev->family == CHIP_TAHITI) ||
1823 (rdev->family == CHIP_PITCAIRN)) {
1824 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1825 switch (reg_offset) {
1826 case 0: /* non-AA compressed depth or any compressed stencil */
1827 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1828 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1829 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1830 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1831 NUM_BANKS(ADDR_SURF_16_BANK) |
1832 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1833 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1834 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1836 case 1: /* 2xAA/4xAA compressed depth only */
1837 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1838 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1839 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1840 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1841 NUM_BANKS(ADDR_SURF_16_BANK) |
1842 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1843 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1844 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1846 case 2: /* 8xAA compressed depth only */
1847 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1848 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1849 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1850 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1851 NUM_BANKS(ADDR_SURF_16_BANK) |
1852 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1853 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1854 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1856 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
1857 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1858 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1859 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1860 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1861 NUM_BANKS(ADDR_SURF_16_BANK) |
1862 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1863 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1864 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1866 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
1867 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1868 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1869 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1870 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1871 NUM_BANKS(ADDR_SURF_16_BANK) |
1872 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1873 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1874 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1876 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
1877 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1878 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1879 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1880 TILE_SPLIT(split_equal_to_row_size) |
1881 NUM_BANKS(ADDR_SURF_16_BANK) |
1882 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1883 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1884 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1886 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
1887 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1888 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1889 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1890 TILE_SPLIT(split_equal_to_row_size) |
1891 NUM_BANKS(ADDR_SURF_16_BANK) |
1892 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1893 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1894 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1896 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
1897 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1898 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1899 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1900 TILE_SPLIT(split_equal_to_row_size) |
1901 NUM_BANKS(ADDR_SURF_16_BANK) |
1902 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1903 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1904 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1906 case 8: /* 1D and 1D Array Surfaces */
1907 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1908 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1909 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1910 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1911 NUM_BANKS(ADDR_SURF_16_BANK) |
1912 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1913 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1914 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1916 case 9: /* Displayable maps. */
1917 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1918 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1919 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1920 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1921 NUM_BANKS(ADDR_SURF_16_BANK) |
1922 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1923 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1924 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1926 case 10: /* Display 8bpp. */
1927 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1928 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1929 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1930 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1931 NUM_BANKS(ADDR_SURF_16_BANK) |
1932 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1933 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1934 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1936 case 11: /* Display 16bpp. */
1937 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1938 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1939 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1940 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1941 NUM_BANKS(ADDR_SURF_16_BANK) |
1942 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1943 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1944 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1946 case 12: /* Display 32bpp. */
1947 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1948 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1949 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1950 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1951 NUM_BANKS(ADDR_SURF_16_BANK) |
1952 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1953 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1954 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1956 case 13: /* Thin. */
1957 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1958 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1959 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1960 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1961 NUM_BANKS(ADDR_SURF_16_BANK) |
1962 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1963 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1964 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1966 case 14: /* Thin 8 bpp. */
1967 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1968 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1969 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1970 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1971 NUM_BANKS(ADDR_SURF_16_BANK) |
1972 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1976 case 15: /* Thin 16 bpp. */
1977 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1978 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1979 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1980 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1981 NUM_BANKS(ADDR_SURF_16_BANK) |
1982 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1983 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1984 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1986 case 16: /* Thin 32 bpp. */
1987 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1988 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1989 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1990 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1991 NUM_BANKS(ADDR_SURF_16_BANK) |
1992 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1993 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1994 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1996 case 17: /* Thin 64 bpp. */
1997 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1998 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1999 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2000 TILE_SPLIT(split_equal_to_row_size) |
2001 NUM_BANKS(ADDR_SURF_16_BANK) |
2002 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2003 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2004 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2006 case 21: /* 8 bpp PRT. */
2007 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2008 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2009 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2010 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2011 NUM_BANKS(ADDR_SURF_16_BANK) |
2012 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2013 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2014 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2016 case 22: /* 16 bpp PRT */
2017 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2018 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2019 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2020 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2021 NUM_BANKS(ADDR_SURF_16_BANK) |
2022 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2023 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2024 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2026 case 23: /* 32 bpp PRT */
2027 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2028 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2029 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2030 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2031 NUM_BANKS(ADDR_SURF_16_BANK) |
2032 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2033 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2034 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2036 case 24: /* 64 bpp PRT */
2037 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2038 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2039 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2040 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2041 NUM_BANKS(ADDR_SURF_16_BANK) |
2042 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2043 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2044 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2046 case 25: /* 128 bpp PRT */
2047 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2048 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2049 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2050 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2051 NUM_BANKS(ADDR_SURF_8_BANK) |
2052 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2053 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2054 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2060 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2061 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2063 } else if ((rdev->family == CHIP_VERDE) ||
2064 (rdev->family == CHIP_OLAND) ||
2065 (rdev->family == CHIP_HAINAN)) {
2066 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2067 switch (reg_offset) {
2068 case 0: /* non-AA compressed depth or any compressed stencil */
2069 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2070 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2071 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2072 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2073 NUM_BANKS(ADDR_SURF_16_BANK) |
2074 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2075 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2076 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2078 case 1: /* 2xAA/4xAA compressed depth only */
2079 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2080 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2081 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2082 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2083 NUM_BANKS(ADDR_SURF_16_BANK) |
2084 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2085 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2086 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2088 case 2: /* 8xAA compressed depth only */
2089 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2090 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2091 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2092 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2093 NUM_BANKS(ADDR_SURF_16_BANK) |
2094 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2095 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2096 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2098 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2099 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2100 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2101 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2102 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2103 NUM_BANKS(ADDR_SURF_16_BANK) |
2104 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2105 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2106 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2108 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2109 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2110 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2111 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2112 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2113 NUM_BANKS(ADDR_SURF_16_BANK) |
2114 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2115 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2116 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2118 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2119 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2120 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2121 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2122 TILE_SPLIT(split_equal_to_row_size) |
2123 NUM_BANKS(ADDR_SURF_16_BANK) |
2124 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2125 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2126 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2128 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2129 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2130 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2131 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2132 TILE_SPLIT(split_equal_to_row_size) |
2133 NUM_BANKS(ADDR_SURF_16_BANK) |
2134 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2135 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2136 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2138 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2139 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2140 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2141 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2142 TILE_SPLIT(split_equal_to_row_size) |
2143 NUM_BANKS(ADDR_SURF_16_BANK) |
2144 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2145 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2146 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2148 case 8: /* 1D and 1D Array Surfaces */
2149 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2150 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2151 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2152 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2153 NUM_BANKS(ADDR_SURF_16_BANK) |
2154 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2155 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2156 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2158 case 9: /* Displayable maps. */
2159 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2160 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2161 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2162 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2163 NUM_BANKS(ADDR_SURF_16_BANK) |
2164 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2165 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2166 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2168 case 10: /* Display 8bpp. */
2169 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2170 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2171 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2172 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2173 NUM_BANKS(ADDR_SURF_16_BANK) |
2174 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2175 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2176 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2178 case 11: /* Display 16bpp. */
2179 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2180 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2181 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2182 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2183 NUM_BANKS(ADDR_SURF_16_BANK) |
2184 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2185 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2186 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2188 case 12: /* Display 32bpp. */
2189 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2190 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2191 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2192 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2193 NUM_BANKS(ADDR_SURF_16_BANK) |
2194 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2195 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2196 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2198 case 13: /* Thin. */
2199 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2200 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2201 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2202 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2203 NUM_BANKS(ADDR_SURF_16_BANK) |
2204 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2205 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2206 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2208 case 14: /* Thin 8 bpp. */
2209 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2210 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2211 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2212 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2213 NUM_BANKS(ADDR_SURF_16_BANK) |
2214 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2215 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2216 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2218 case 15: /* Thin 16 bpp. */
2219 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2220 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2221 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2222 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2223 NUM_BANKS(ADDR_SURF_16_BANK) |
2224 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2225 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2226 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2228 case 16: /* Thin 32 bpp. */
2229 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2230 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2231 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2232 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2233 NUM_BANKS(ADDR_SURF_16_BANK) |
2234 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2235 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2236 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2238 case 17: /* Thin 64 bpp. */
2239 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2240 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2241 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2242 TILE_SPLIT(split_equal_to_row_size) |
2243 NUM_BANKS(ADDR_SURF_16_BANK) |
2244 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2245 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2246 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2248 case 21: /* 8 bpp PRT. */
2249 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2250 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2251 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2252 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2253 NUM_BANKS(ADDR_SURF_16_BANK) |
2254 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2255 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2256 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2258 case 22: /* 16 bpp PRT */
2259 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2260 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2261 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2262 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2263 NUM_BANKS(ADDR_SURF_16_BANK) |
2264 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2268 case 23: /* 32 bpp PRT */
2269 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2270 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2271 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2272 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2273 NUM_BANKS(ADDR_SURF_16_BANK) |
2274 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2275 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2276 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2278 case 24: /* 64 bpp PRT */
2279 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2280 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2281 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2282 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2283 NUM_BANKS(ADDR_SURF_16_BANK) |
2284 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2285 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2286 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2288 case 25: /* 128 bpp PRT */
2289 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2290 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2291 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2292 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2293 NUM_BANKS(ADDR_SURF_8_BANK) |
2294 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2302 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2303 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2306 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2309 static void si_select_se_sh(struct radeon_device *rdev,
2310 u32 se_num, u32 sh_num)
2312 u32 data = INSTANCE_BROADCAST_WRITES;
2314 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2315 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2316 else if (se_num == 0xffffffff)
2317 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2318 else if (sh_num == 0xffffffff)
2319 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2321 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2322 WREG32(GRBM_GFX_INDEX, data);
2325 static u32 si_create_bitmask(u32 bit_width)
2329 for (i = 0; i < bit_width; i++) {
2336 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2340 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2342 data &= INACTIVE_CUS_MASK;
2345 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2347 data >>= INACTIVE_CUS_SHIFT;
2349 mask = si_create_bitmask(cu_per_sh);
2351 return ~data & mask;
2354 static void si_setup_spi(struct radeon_device *rdev,
2355 u32 se_num, u32 sh_per_se,
2359 u32 data, mask, active_cu;
2361 for (i = 0; i < se_num; i++) {
2362 for (j = 0; j < sh_per_se; j++) {
2363 si_select_se_sh(rdev, i, j);
2364 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2365 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2368 for (k = 0; k < 16; k++) {
2370 if (active_cu & mask) {
2372 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2378 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2381 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2382 u32 max_rb_num, u32 se_num,
2387 data = RREG32(CC_RB_BACKEND_DISABLE);
2389 data &= BACKEND_DISABLE_MASK;
2392 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2394 data >>= BACKEND_DISABLE_SHIFT;
2396 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2401 static void si_setup_rb(struct radeon_device *rdev,
2402 u32 se_num, u32 sh_per_se,
2407 u32 disabled_rbs = 0;
2408 u32 enabled_rbs = 0;
2410 for (i = 0; i < se_num; i++) {
2411 for (j = 0; j < sh_per_se; j++) {
2412 si_select_se_sh(rdev, i, j);
2413 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2414 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2417 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2420 for (i = 0; i < max_rb_num; i++) {
2421 if (!(disabled_rbs & mask))
2422 enabled_rbs |= mask;
2426 for (i = 0; i < se_num; i++) {
2427 si_select_se_sh(rdev, i, 0xffffffff);
2429 for (j = 0; j < sh_per_se; j++) {
2430 switch (enabled_rbs & 3) {
2432 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2435 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2439 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2444 WREG32(PA_SC_RASTER_CONFIG, data);
2446 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2449 static void si_gpu_init(struct radeon_device *rdev)
2451 u32 gb_addr_config = 0;
2452 u32 mc_shared_chmap, mc_arb_ramcfg;
2454 u32 hdp_host_path_cntl;
2458 switch (rdev->family) {
2460 rdev->config.si.max_shader_engines = 2;
2461 rdev->config.si.max_tile_pipes = 12;
2462 rdev->config.si.max_cu_per_sh = 8;
2463 rdev->config.si.max_sh_per_se = 2;
2464 rdev->config.si.max_backends_per_se = 4;
2465 rdev->config.si.max_texture_channel_caches = 12;
2466 rdev->config.si.max_gprs = 256;
2467 rdev->config.si.max_gs_threads = 32;
2468 rdev->config.si.max_hw_contexts = 8;
2470 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2471 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2472 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2473 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2474 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2477 rdev->config.si.max_shader_engines = 2;
2478 rdev->config.si.max_tile_pipes = 8;
2479 rdev->config.si.max_cu_per_sh = 5;
2480 rdev->config.si.max_sh_per_se = 2;
2481 rdev->config.si.max_backends_per_se = 4;
2482 rdev->config.si.max_texture_channel_caches = 8;
2483 rdev->config.si.max_gprs = 256;
2484 rdev->config.si.max_gs_threads = 32;
2485 rdev->config.si.max_hw_contexts = 8;
2487 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2488 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2489 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2490 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2491 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2495 rdev->config.si.max_shader_engines = 1;
2496 rdev->config.si.max_tile_pipes = 4;
2497 rdev->config.si.max_cu_per_sh = 2;
2498 rdev->config.si.max_sh_per_se = 2;
2499 rdev->config.si.max_backends_per_se = 4;
2500 rdev->config.si.max_texture_channel_caches = 4;
2501 rdev->config.si.max_gprs = 256;
2502 rdev->config.si.max_gs_threads = 32;
2503 rdev->config.si.max_hw_contexts = 8;
2505 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2506 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2507 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2508 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2509 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2512 rdev->config.si.max_shader_engines = 1;
2513 rdev->config.si.max_tile_pipes = 4;
2514 rdev->config.si.max_cu_per_sh = 6;
2515 rdev->config.si.max_sh_per_se = 1;
2516 rdev->config.si.max_backends_per_se = 2;
2517 rdev->config.si.max_texture_channel_caches = 4;
2518 rdev->config.si.max_gprs = 256;
2519 rdev->config.si.max_gs_threads = 16;
2520 rdev->config.si.max_hw_contexts = 8;
2522 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2523 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2524 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2525 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2526 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2529 rdev->config.si.max_shader_engines = 1;
2530 rdev->config.si.max_tile_pipes = 4;
2531 rdev->config.si.max_cu_per_sh = 5;
2532 rdev->config.si.max_sh_per_se = 1;
2533 rdev->config.si.max_backends_per_se = 1;
2534 rdev->config.si.max_texture_channel_caches = 2;
2535 rdev->config.si.max_gprs = 256;
2536 rdev->config.si.max_gs_threads = 16;
2537 rdev->config.si.max_hw_contexts = 8;
2539 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2540 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2541 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2542 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2543 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2547 /* Initialize HDP */
2548 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2549 WREG32((0x2c14 + j), 0x00000000);
2550 WREG32((0x2c18 + j), 0x00000000);
2551 WREG32((0x2c1c + j), 0x00000000);
2552 WREG32((0x2c20 + j), 0x00000000);
2553 WREG32((0x2c24 + j), 0x00000000);
2556 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2558 evergreen_fix_pci_max_read_req_size(rdev);
2560 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2562 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2563 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2565 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
2566 rdev->config.si.mem_max_burst_length_bytes = 256;
2567 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2568 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2569 if (rdev->config.si.mem_row_size_in_kb > 4)
2570 rdev->config.si.mem_row_size_in_kb = 4;
2571 /* XXX use MC settings? */
2572 rdev->config.si.shader_engine_tile_size = 32;
2573 rdev->config.si.num_gpus = 1;
2574 rdev->config.si.multi_gpu_tile_size = 64;
2576 /* fix up row size */
2577 gb_addr_config &= ~ROW_SIZE_MASK;
2578 switch (rdev->config.si.mem_row_size_in_kb) {
2581 gb_addr_config |= ROW_SIZE(0);
2584 gb_addr_config |= ROW_SIZE(1);
2587 gb_addr_config |= ROW_SIZE(2);
2591 /* setup tiling info dword. gb_addr_config is not adequate since it does
2592 * not have bank info, so create a custom tiling dword.
2593 * bits 3:0 num_pipes
2594 * bits 7:4 num_banks
2595 * bits 11:8 group_size
2596 * bits 15:12 row_size
2598 rdev->config.si.tile_config = 0;
2599 switch (rdev->config.si.num_tile_pipes) {
2601 rdev->config.si.tile_config |= (0 << 0);
2604 rdev->config.si.tile_config |= (1 << 0);
2607 rdev->config.si.tile_config |= (2 << 0);
2611 /* XXX what about 12? */
2612 rdev->config.si.tile_config |= (3 << 0);
2615 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
2616 case 0: /* four banks */
2617 rdev->config.si.tile_config |= 0 << 4;
2619 case 1: /* eight banks */
2620 rdev->config.si.tile_config |= 1 << 4;
2622 case 2: /* sixteen banks */
2624 rdev->config.si.tile_config |= 2 << 4;
2627 rdev->config.si.tile_config |=
2628 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2629 rdev->config.si.tile_config |=
2630 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2632 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2633 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
2634 WREG32(DMIF_ADDR_CALC, gb_addr_config);
2635 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2636 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
2637 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
2638 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2639 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2640 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
2642 si_tiling_mode_table_init(rdev);
2644 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
2645 rdev->config.si.max_sh_per_se,
2646 rdev->config.si.max_backends_per_se);
2648 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
2649 rdev->config.si.max_sh_per_se,
2650 rdev->config.si.max_cu_per_sh);
2653 /* set HW defaults for 3D engine */
2654 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2655 ROQ_IB2_START(0x2b)));
2656 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2658 sx_debug_1 = RREG32(SX_DEBUG_1);
2659 WREG32(SX_DEBUG_1, sx_debug_1);
2661 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2663 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
2664 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
2665 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
2666 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
2668 WREG32(VGT_NUM_INSTANCES, 1);
2670 WREG32(CP_PERFMON_CNTL, 0);
2672 WREG32(SQ_CONFIG, 0);
2674 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2675 FORCE_EOV_MAX_REZ_CNT(255)));
2677 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2678 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2680 WREG32(VGT_GS_VERTEX_REUSE, 16);
2681 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2683 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
2684 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
2685 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
2686 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
2687 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
2688 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
2689 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
2690 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
2692 tmp = RREG32(HDP_MISC_CNTL);
2693 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2694 WREG32(HDP_MISC_CNTL, tmp);
2696 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2697 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2699 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2705 * GPU scratch registers helpers function.
2707 static void si_scratch_init(struct radeon_device *rdev)
2711 rdev->scratch.num_reg = 7;
2712 rdev->scratch.reg_base = SCRATCH_REG0;
2713 for (i = 0; i < rdev->scratch.num_reg; i++) {
2714 rdev->scratch.free[i] = true;
2715 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2719 void si_fence_ring_emit(struct radeon_device *rdev,
2720 struct radeon_fence *fence)
2722 struct radeon_ring *ring = &rdev->ring[fence->ring];
2723 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2725 /* flush read cache over gart */
2726 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2727 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
2728 radeon_ring_write(ring, 0);
2729 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2730 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2731 PACKET3_TC_ACTION_ENA |
2732 PACKET3_SH_KCACHE_ACTION_ENA |
2733 PACKET3_SH_ICACHE_ACTION_ENA);
2734 radeon_ring_write(ring, 0xFFFFFFFF);
2735 radeon_ring_write(ring, 0);
2736 radeon_ring_write(ring, 10); /* poll interval */
2737 /* EVENT_WRITE_EOP - flush caches, send int */
2738 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2739 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
2740 radeon_ring_write(ring, addr & 0xffffffff);
2741 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2742 radeon_ring_write(ring, fence->seq);
2743 radeon_ring_write(ring, 0);
2749 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2751 struct radeon_ring *ring = &rdev->ring[ib->ring];
2754 if (ib->is_const_ib) {
2755 /* set switch buffer packet before const IB */
2756 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2757 radeon_ring_write(ring, 0);
2759 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
2762 if (ring->rptr_save_reg) {
2763 next_rptr = ring->wptr + 3 + 4 + 8;
2764 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2765 radeon_ring_write(ring, ((ring->rptr_save_reg -
2766 PACKET3_SET_CONFIG_REG_START) >> 2));
2767 radeon_ring_write(ring, next_rptr);
2768 } else if (rdev->wb.enabled) {
2769 next_rptr = ring->wptr + 5 + 4 + 8;
2770 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2771 radeon_ring_write(ring, (1 << 8));
2772 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2773 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
2774 radeon_ring_write(ring, next_rptr);
2777 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
2780 radeon_ring_write(ring, header);
2781 radeon_ring_write(ring,
2785 (ib->gpu_addr & 0xFFFFFFFC));
2786 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
2787 radeon_ring_write(ring, ib->length_dw |
2788 (ib->vm ? (ib->vm->id << 24) : 0));
2790 if (!ib->is_const_ib) {
2791 /* flush read cache over gart for this vmid */
2792 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2793 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
2794 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
2795 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2796 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2797 PACKET3_TC_ACTION_ENA |
2798 PACKET3_SH_KCACHE_ACTION_ENA |
2799 PACKET3_SH_ICACHE_ACTION_ENA);
2800 radeon_ring_write(ring, 0xFFFFFFFF);
2801 radeon_ring_write(ring, 0);
2802 radeon_ring_write(ring, 10); /* poll interval */
2809 static void si_cp_enable(struct radeon_device *rdev, bool enable)
2812 WREG32(CP_ME_CNTL, 0);
2814 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2815 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
2816 WREG32(SCRATCH_UMSK, 0);
2817 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2818 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2819 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2824 static int si_cp_load_microcode(struct radeon_device *rdev)
2826 const __be32 *fw_data;
2829 if (!rdev->me_fw || !rdev->pfp_fw)
2832 si_cp_enable(rdev, false);
2835 fw_data = (const __be32 *)rdev->pfp_fw->data;
2836 WREG32(CP_PFP_UCODE_ADDR, 0);
2837 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
2838 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2839 WREG32(CP_PFP_UCODE_ADDR, 0);
2842 fw_data = (const __be32 *)rdev->ce_fw->data;
2843 WREG32(CP_CE_UCODE_ADDR, 0);
2844 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
2845 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
2846 WREG32(CP_CE_UCODE_ADDR, 0);
2849 fw_data = (const __be32 *)rdev->me_fw->data;
2850 WREG32(CP_ME_RAM_WADDR, 0);
2851 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
2852 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2853 WREG32(CP_ME_RAM_WADDR, 0);
2855 WREG32(CP_PFP_UCODE_ADDR, 0);
2856 WREG32(CP_CE_UCODE_ADDR, 0);
2857 WREG32(CP_ME_RAM_WADDR, 0);
2858 WREG32(CP_ME_RAM_RADDR, 0);
2862 static int si_cp_start(struct radeon_device *rdev)
2864 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2867 r = radeon_ring_lock(rdev, ring, 7 + 4);
2869 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2873 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2874 radeon_ring_write(ring, 0x1);
2875 radeon_ring_write(ring, 0x0);
2876 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
2877 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2878 radeon_ring_write(ring, 0);
2879 radeon_ring_write(ring, 0);
2881 /* init the CE partitions */
2882 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2883 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2884 radeon_ring_write(ring, 0xc000);
2885 radeon_ring_write(ring, 0xe000);
2886 radeon_ring_unlock_commit(rdev, ring);
2888 si_cp_enable(rdev, true);
2890 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
2892 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2896 /* setup clear context state */
2897 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2898 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2900 for (i = 0; i < si_default_size; i++)
2901 radeon_ring_write(ring, si_default_state[i]);
2903 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2904 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2906 /* set clear context state */
2907 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2908 radeon_ring_write(ring, 0);
2910 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2911 radeon_ring_write(ring, 0x00000316);
2912 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2913 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
2915 radeon_ring_unlock_commit(rdev, ring);
2917 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
2918 ring = &rdev->ring[i];
2919 r = radeon_ring_lock(rdev, ring, 2);
2921 /* clear the compute context state */
2922 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
2923 radeon_ring_write(ring, 0);
2925 radeon_ring_unlock_commit(rdev, ring);
2931 static void si_cp_fini(struct radeon_device *rdev)
2933 struct radeon_ring *ring;
2934 si_cp_enable(rdev, false);
2936 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2937 radeon_ring_fini(rdev, ring);
2938 radeon_scratch_free(rdev, ring->rptr_save_reg);
2940 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
2941 radeon_ring_fini(rdev, ring);
2942 radeon_scratch_free(rdev, ring->rptr_save_reg);
2944 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
2945 radeon_ring_fini(rdev, ring);
2946 radeon_scratch_free(rdev, ring->rptr_save_reg);
2949 static int si_cp_resume(struct radeon_device *rdev)
2951 struct radeon_ring *ring;
2956 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2957 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2962 RREG32(GRBM_SOFT_RESET);
2964 WREG32(GRBM_SOFT_RESET, 0);
2965 RREG32(GRBM_SOFT_RESET);
2967 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2968 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2970 /* Set the write pointer delay */
2971 WREG32(CP_RB_WPTR_DELAY, 0);
2973 WREG32(CP_DEBUG, 0);
2974 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2976 /* ring 0 - compute and gfx */
2977 /* Set ring buffer size */
2978 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2979 rb_bufsz = drm_order(ring->ring_size / 8);
2980 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2982 tmp |= BUF_SWAP_32BIT;
2984 WREG32(CP_RB0_CNTL, tmp);
2986 /* Initialize the ring buffer's read and write pointers */
2987 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
2989 WREG32(CP_RB0_WPTR, ring->wptr);
2991 /* set the wb address whether it's enabled or not */
2992 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
2993 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2995 if (rdev->wb.enabled)
2996 WREG32(SCRATCH_UMSK, 0xff);
2998 tmp |= RB_NO_UPDATE;
2999 WREG32(SCRATCH_UMSK, 0);
3003 WREG32(CP_RB0_CNTL, tmp);
3005 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3007 ring->rptr = RREG32(CP_RB0_RPTR);
3009 /* ring1 - compute only */
3010 /* Set ring buffer size */
3011 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3012 rb_bufsz = drm_order(ring->ring_size / 8);
3013 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3015 tmp |= BUF_SWAP_32BIT;
3017 WREG32(CP_RB1_CNTL, tmp);
3019 /* Initialize the ring buffer's read and write pointers */
3020 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3022 WREG32(CP_RB1_WPTR, ring->wptr);
3024 /* set the wb address whether it's enabled or not */
3025 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3026 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3029 WREG32(CP_RB1_CNTL, tmp);
3031 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3033 ring->rptr = RREG32(CP_RB1_RPTR);
3035 /* ring2 - compute only */
3036 /* Set ring buffer size */
3037 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3038 rb_bufsz = drm_order(ring->ring_size / 8);
3039 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3041 tmp |= BUF_SWAP_32BIT;
3043 WREG32(CP_RB2_CNTL, tmp);
3045 /* Initialize the ring buffer's read and write pointers */
3046 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3048 WREG32(CP_RB2_WPTR, ring->wptr);
3050 /* set the wb address whether it's enabled or not */
3051 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3052 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3055 WREG32(CP_RB2_CNTL, tmp);
3057 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3059 ring->rptr = RREG32(CP_RB2_RPTR);
3061 /* start the rings */
3063 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3064 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3065 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3066 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3068 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3069 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3070 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3073 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3075 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3077 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3079 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3085 static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3091 tmp = RREG32(GRBM_STATUS);
3092 if (tmp & (PA_BUSY | SC_BUSY |
3093 BCI_BUSY | SX_BUSY |
3094 TA_BUSY | VGT_BUSY |
3096 GDS_BUSY | SPI_BUSY |
3097 IA_BUSY | IA_BUSY_NO_DMA))
3098 reset_mask |= RADEON_RESET_GFX;
3100 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3101 CP_BUSY | CP_COHERENCY_BUSY))
3102 reset_mask |= RADEON_RESET_CP;
3104 if (tmp & GRBM_EE_BUSY)
3105 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3108 tmp = RREG32(GRBM_STATUS2);
3109 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3110 reset_mask |= RADEON_RESET_RLC;
3112 /* DMA_STATUS_REG 0 */
3113 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3114 if (!(tmp & DMA_IDLE))
3115 reset_mask |= RADEON_RESET_DMA;
3117 /* DMA_STATUS_REG 1 */
3118 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3119 if (!(tmp & DMA_IDLE))
3120 reset_mask |= RADEON_RESET_DMA1;
3123 tmp = RREG32(SRBM_STATUS2);
3125 reset_mask |= RADEON_RESET_DMA;
3127 if (tmp & DMA1_BUSY)
3128 reset_mask |= RADEON_RESET_DMA1;
3131 tmp = RREG32(SRBM_STATUS);
3134 reset_mask |= RADEON_RESET_IH;
3137 reset_mask |= RADEON_RESET_SEM;
3139 if (tmp & GRBM_RQ_PENDING)
3140 reset_mask |= RADEON_RESET_GRBM;
3143 reset_mask |= RADEON_RESET_VMC;
3145 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3146 MCC_BUSY | MCD_BUSY))
3147 reset_mask |= RADEON_RESET_MC;
3149 if (evergreen_is_display_hung(rdev))
3150 reset_mask |= RADEON_RESET_DISPLAY;
3153 tmp = RREG32(VM_L2_STATUS);
3155 reset_mask |= RADEON_RESET_VMC;
3157 /* Skip MC reset as it's mostly likely not hung, just busy */
3158 if (reset_mask & RADEON_RESET_MC) {
3159 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3160 reset_mask &= ~RADEON_RESET_MC;
3166 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3168 struct evergreen_mc_save save;
3169 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3172 if (reset_mask == 0)
3175 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3177 evergreen_print_gpu_status_regs(rdev);
3178 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3179 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3180 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3181 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3183 /* Disable CP parsing/prefetching */
3184 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3186 if (reset_mask & RADEON_RESET_DMA) {
3188 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3189 tmp &= ~DMA_RB_ENABLE;
3190 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3192 if (reset_mask & RADEON_RESET_DMA1) {
3194 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3195 tmp &= ~DMA_RB_ENABLE;
3196 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3201 evergreen_mc_stop(rdev, &save);
3202 if (evergreen_mc_wait_for_idle(rdev)) {
3203 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3206 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3207 grbm_soft_reset = SOFT_RESET_CB |
3221 if (reset_mask & RADEON_RESET_CP) {
3222 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3224 srbm_soft_reset |= SOFT_RESET_GRBM;
3227 if (reset_mask & RADEON_RESET_DMA)
3228 srbm_soft_reset |= SOFT_RESET_DMA;
3230 if (reset_mask & RADEON_RESET_DMA1)
3231 srbm_soft_reset |= SOFT_RESET_DMA1;
3233 if (reset_mask & RADEON_RESET_DISPLAY)
3234 srbm_soft_reset |= SOFT_RESET_DC;
3236 if (reset_mask & RADEON_RESET_RLC)
3237 grbm_soft_reset |= SOFT_RESET_RLC;
3239 if (reset_mask & RADEON_RESET_SEM)
3240 srbm_soft_reset |= SOFT_RESET_SEM;
3242 if (reset_mask & RADEON_RESET_IH)
3243 srbm_soft_reset |= SOFT_RESET_IH;
3245 if (reset_mask & RADEON_RESET_GRBM)
3246 srbm_soft_reset |= SOFT_RESET_GRBM;
3248 if (reset_mask & RADEON_RESET_VMC)
3249 srbm_soft_reset |= SOFT_RESET_VMC;
3251 if (reset_mask & RADEON_RESET_MC)
3252 srbm_soft_reset |= SOFT_RESET_MC;
3254 if (grbm_soft_reset) {
3255 tmp = RREG32(GRBM_SOFT_RESET);
3256 tmp |= grbm_soft_reset;
3257 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3258 WREG32(GRBM_SOFT_RESET, tmp);
3259 tmp = RREG32(GRBM_SOFT_RESET);
3263 tmp &= ~grbm_soft_reset;
3264 WREG32(GRBM_SOFT_RESET, tmp);
3265 tmp = RREG32(GRBM_SOFT_RESET);
3268 if (srbm_soft_reset) {
3269 tmp = RREG32(SRBM_SOFT_RESET);
3270 tmp |= srbm_soft_reset;
3271 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3272 WREG32(SRBM_SOFT_RESET, tmp);
3273 tmp = RREG32(SRBM_SOFT_RESET);
3277 tmp &= ~srbm_soft_reset;
3278 WREG32(SRBM_SOFT_RESET, tmp);
3279 tmp = RREG32(SRBM_SOFT_RESET);
3282 /* Wait a little for things to settle down */
3285 evergreen_mc_resume(rdev, &save);
3288 evergreen_print_gpu_status_regs(rdev);
3291 int si_asic_reset(struct radeon_device *rdev)
3295 reset_mask = si_gpu_check_soft_reset(rdev);
3298 r600_set_bios_scratch_engine_hung(rdev, true);
3300 si_gpu_soft_reset(rdev, reset_mask);
3302 reset_mask = si_gpu_check_soft_reset(rdev);
3305 r600_set_bios_scratch_engine_hung(rdev, false);
3311 * si_gfx_is_lockup - Check if the GFX engine is locked up
3313 * @rdev: radeon_device pointer
3314 * @ring: radeon_ring structure holding ring information
3316 * Check if the GFX engine is locked up.
3317 * Returns true if the engine appears to be locked up, false if not.
3319 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3321 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3323 if (!(reset_mask & (RADEON_RESET_GFX |
3324 RADEON_RESET_COMPUTE |
3325 RADEON_RESET_CP))) {
3326 radeon_ring_lockup_update(ring);
3329 /* force CP activities */
3330 radeon_ring_force_activity(rdev, ring);
3331 return radeon_ring_test_lockup(rdev, ring);
3335 * si_dma_is_lockup - Check if the DMA engine is locked up
3337 * @rdev: radeon_device pointer
3338 * @ring: radeon_ring structure holding ring information
3340 * Check if the async DMA engine is locked up.
3341 * Returns true if the engine appears to be locked up, false if not.
3343 bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3345 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3348 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3349 mask = RADEON_RESET_DMA;
3351 mask = RADEON_RESET_DMA1;
3353 if (!(reset_mask & mask)) {
3354 radeon_ring_lockup_update(ring);
3357 /* force ring activities */
3358 radeon_ring_force_activity(rdev, ring);
3359 return radeon_ring_test_lockup(rdev, ring);
3363 static void si_mc_program(struct radeon_device *rdev)
3365 struct evergreen_mc_save save;
3369 /* Initialize HDP */
3370 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3371 WREG32((0x2c14 + j), 0x00000000);
3372 WREG32((0x2c18 + j), 0x00000000);
3373 WREG32((0x2c1c + j), 0x00000000);
3374 WREG32((0x2c20 + j), 0x00000000);
3375 WREG32((0x2c24 + j), 0x00000000);
3377 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3379 evergreen_mc_stop(rdev, &save);
3380 if (radeon_mc_wait_for_idle(rdev)) {
3381 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3383 if (!ASIC_IS_NODCE(rdev))
3384 /* Lockout access through VGA aperture*/
3385 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3386 /* Update configuration */
3387 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3388 rdev->mc.vram_start >> 12);
3389 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3390 rdev->mc.vram_end >> 12);
3391 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3392 rdev->vram_scratch.gpu_addr >> 12);
3393 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3394 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3395 WREG32(MC_VM_FB_LOCATION, tmp);
3396 /* XXX double check these! */
3397 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3398 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3399 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3400 WREG32(MC_VM_AGP_BASE, 0);
3401 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3402 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3403 if (radeon_mc_wait_for_idle(rdev)) {
3404 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3406 evergreen_mc_resume(rdev, &save);
3407 if (!ASIC_IS_NODCE(rdev)) {
3408 /* we need to own VRAM, so turn off the VGA renderer here
3409 * to stop it overwriting our objects */
3410 rv515_vga_render_disable(rdev);
3414 static void si_vram_gtt_location(struct radeon_device *rdev,
3415 struct radeon_mc *mc)
3417 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3418 /* leave room for at least 1024M GTT */
3419 dev_warn(rdev->dev, "limiting VRAM\n");
3420 mc->real_vram_size = 0xFFC0000000ULL;
3421 mc->mc_vram_size = 0xFFC0000000ULL;
3423 radeon_vram_location(rdev, &rdev->mc, 0);
3424 rdev->mc.gtt_base_align = 0;
3425 radeon_gtt_location(rdev, mc);
3428 static int si_mc_init(struct radeon_device *rdev)
3431 int chansize, numchan;
3433 /* Get VRAM informations */
3434 rdev->mc.vram_is_ddr = true;
3435 tmp = RREG32(MC_ARB_RAMCFG);
3436 if (tmp & CHANSIZE_OVERRIDE) {
3438 } else if (tmp & CHANSIZE_MASK) {
3443 tmp = RREG32(MC_SHARED_CHMAP);
3444 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3474 rdev->mc.vram_width = numchan * chansize;
3475 /* Could aper size report 0 ? */
3476 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3477 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3478 /* size in MB on si */
3479 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3480 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3481 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3482 si_vram_gtt_location(rdev, &rdev->mc);
3483 radeon_update_bandwidth_info(rdev);
3491 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3493 /* flush hdp cache */
3494 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3496 /* bits 0-15 are the VM contexts0-15 */
3497 WREG32(VM_INVALIDATE_REQUEST, 1);
3500 static int si_pcie_gart_enable(struct radeon_device *rdev)
3504 if (rdev->gart.robj == NULL) {
3505 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3508 r = radeon_gart_table_vram_pin(rdev);
3511 radeon_gart_restore(rdev);
3512 /* Setup TLB control */
3513 WREG32(MC_VM_MX_L1_TLB_CNTL,
3516 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3517 ENABLE_ADVANCED_DRIVER_MODEL |
3518 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3519 /* Setup L2 cache */
3520 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3521 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3522 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3523 EFFECTIVE_L2_QUEUE_SIZE(7) |
3524 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3525 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3526 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3527 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3528 /* setup context0 */
3529 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3530 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3531 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3532 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3533 (u32)(rdev->dummy_page.addr >> 12));
3534 WREG32(VM_CONTEXT0_CNTL2, 0);
3535 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3536 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3542 /* empty context1-15 */
3543 /* set vm size, must be a multiple of 4 */
3544 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
3545 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
3546 /* Assign the pt base to something valid for now; the pts used for
3547 * the VMs are determined by the application and setup and assigned
3548 * on the fly in the vm part of radeon_gart.c
3550 for (i = 1; i < 16; i++) {
3552 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3553 rdev->gart.table_addr >> 12);
3555 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3556 rdev->gart.table_addr >> 12);
3559 /* enable context1-15 */
3560 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3561 (u32)(rdev->dummy_page.addr >> 12));
3562 WREG32(VM_CONTEXT1_CNTL2, 4);
3563 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
3564 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3565 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3566 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3567 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3568 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3569 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3570 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3571 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3572 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3573 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3574 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3575 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
3577 si_pcie_gart_tlb_flush(rdev);
3578 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3579 (unsigned)(rdev->mc.gtt_size >> 20),
3580 (unsigned long long)rdev->gart.table_addr);
3581 rdev->gart.ready = true;
3585 static void si_pcie_gart_disable(struct radeon_device *rdev)
3587 /* Disable all tables */
3588 WREG32(VM_CONTEXT0_CNTL, 0);
3589 WREG32(VM_CONTEXT1_CNTL, 0);
3590 /* Setup TLB control */
3591 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3592 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3593 /* Setup L2 cache */
3594 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3595 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3596 EFFECTIVE_L2_QUEUE_SIZE(7) |
3597 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3598 WREG32(VM_L2_CNTL2, 0);
3599 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3600 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3601 radeon_gart_table_vram_unpin(rdev);
3604 static void si_pcie_gart_fini(struct radeon_device *rdev)
3606 si_pcie_gart_disable(rdev);
3607 radeon_gart_table_vram_free(rdev);
3608 radeon_gart_fini(rdev);
3612 static bool si_vm_reg_valid(u32 reg)
3614 /* context regs are fine */
3618 /* check config regs */
3620 case GRBM_GFX_INDEX:
3621 case CP_STRMOUT_CNTL:
3622 case VGT_VTX_VECT_EJECT_REG:
3623 case VGT_CACHE_INVALIDATION:
3624 case VGT_ESGS_RING_SIZE:
3625 case VGT_GSVS_RING_SIZE:
3626 case VGT_GS_VERTEX_REUSE:
3627 case VGT_PRIMITIVE_TYPE:
3628 case VGT_INDEX_TYPE:
3629 case VGT_NUM_INDICES:
3630 case VGT_NUM_INSTANCES:
3631 case VGT_TF_RING_SIZE:
3632 case VGT_HS_OFFCHIP_PARAM:
3633 case VGT_TF_MEMORY_BASE:
3635 case PA_SU_LINE_STIPPLE_VALUE:
3636 case PA_SC_LINE_STIPPLE_STATE:
3639 case SPI_STATIC_THREAD_MGMT_1:
3640 case SPI_STATIC_THREAD_MGMT_2:
3641 case SPI_STATIC_THREAD_MGMT_3:
3642 case SPI_PS_MAX_WAVE_ID:
3643 case SPI_CONFIG_CNTL:
3644 case SPI_CONFIG_CNTL_1:
3648 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
3653 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
3654 u32 *ib, struct radeon_cs_packet *pkt)
3656 switch (pkt->opcode) {
3658 case PACKET3_SET_BASE:
3659 case PACKET3_SET_CE_DE_COUNTERS:
3660 case PACKET3_LOAD_CONST_RAM:
3661 case PACKET3_WRITE_CONST_RAM:
3662 case PACKET3_WRITE_CONST_RAM_OFFSET:
3663 case PACKET3_DUMP_CONST_RAM:
3664 case PACKET3_INCREMENT_CE_COUNTER:
3665 case PACKET3_WAIT_ON_DE_COUNTER:
3666 case PACKET3_CE_WRITE:
3669 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
3675 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
3676 u32 *ib, struct radeon_cs_packet *pkt)
3678 u32 idx = pkt->idx + 1;
3679 u32 idx_value = ib[idx];
3680 u32 start_reg, end_reg, reg, i;
3683 switch (pkt->opcode) {
3685 case PACKET3_SET_BASE:
3686 case PACKET3_CLEAR_STATE:
3687 case PACKET3_INDEX_BUFFER_SIZE:
3688 case PACKET3_DISPATCH_DIRECT:
3689 case PACKET3_DISPATCH_INDIRECT:
3690 case PACKET3_ALLOC_GDS:
3691 case PACKET3_WRITE_GDS_RAM:
3692 case PACKET3_ATOMIC_GDS:
3693 case PACKET3_ATOMIC:
3694 case PACKET3_OCCLUSION_QUERY:
3695 case PACKET3_SET_PREDICATION:
3696 case PACKET3_COND_EXEC:
3697 case PACKET3_PRED_EXEC:
3698 case PACKET3_DRAW_INDIRECT:
3699 case PACKET3_DRAW_INDEX_INDIRECT:
3700 case PACKET3_INDEX_BASE:
3701 case PACKET3_DRAW_INDEX_2:
3702 case PACKET3_CONTEXT_CONTROL:
3703 case PACKET3_INDEX_TYPE:
3704 case PACKET3_DRAW_INDIRECT_MULTI:
3705 case PACKET3_DRAW_INDEX_AUTO:
3706 case PACKET3_DRAW_INDEX_IMMD:
3707 case PACKET3_NUM_INSTANCES:
3708 case PACKET3_DRAW_INDEX_MULTI_AUTO:
3709 case PACKET3_STRMOUT_BUFFER_UPDATE:
3710 case PACKET3_DRAW_INDEX_OFFSET_2:
3711 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
3712 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
3713 case PACKET3_MPEG_INDEX:
3714 case PACKET3_WAIT_REG_MEM:
3715 case PACKET3_MEM_WRITE:
3716 case PACKET3_PFP_SYNC_ME:
3717 case PACKET3_SURFACE_SYNC:
3718 case PACKET3_EVENT_WRITE:
3719 case PACKET3_EVENT_WRITE_EOP:
3720 case PACKET3_EVENT_WRITE_EOS:
3721 case PACKET3_SET_CONTEXT_REG:
3722 case PACKET3_SET_CONTEXT_REG_INDIRECT:
3723 case PACKET3_SET_SH_REG:
3724 case PACKET3_SET_SH_REG_OFFSET:
3725 case PACKET3_INCREMENT_DE_COUNTER:
3726 case PACKET3_WAIT_ON_CE_COUNTER:
3727 case PACKET3_WAIT_ON_AVAIL_BUFFER:
3728 case PACKET3_ME_WRITE:
3730 case PACKET3_COPY_DATA:
3731 if ((idx_value & 0xf00) == 0) {
3732 reg = ib[idx + 3] * 4;
3733 if (!si_vm_reg_valid(reg))
3737 case PACKET3_WRITE_DATA:
3738 if ((idx_value & 0xf00) == 0) {
3739 start_reg = ib[idx + 1] * 4;
3740 if (idx_value & 0x10000) {
3741 if (!si_vm_reg_valid(start_reg))
3744 for (i = 0; i < (pkt->count - 2); i++) {
3745 reg = start_reg + (4 * i);
3746 if (!si_vm_reg_valid(reg))
3752 case PACKET3_COND_WRITE:
3753 if (idx_value & 0x100) {
3754 reg = ib[idx + 5] * 4;
3755 if (!si_vm_reg_valid(reg))
3759 case PACKET3_COPY_DW:
3760 if (idx_value & 0x2) {
3761 reg = ib[idx + 3] * 4;
3762 if (!si_vm_reg_valid(reg))
3766 case PACKET3_SET_CONFIG_REG:
3767 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
3768 end_reg = 4 * pkt->count + start_reg - 4;
3769 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
3770 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
3771 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
3772 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
3775 for (i = 0; i < pkt->count; i++) {
3776 reg = start_reg + (4 * i);
3777 if (!si_vm_reg_valid(reg))
3781 case PACKET3_CP_DMA:
3782 command = ib[idx + 4];
3784 if (command & PACKET3_CP_DMA_CMD_SAS) {
3785 /* src address space is register */
3786 if (((info & 0x60000000) >> 29) == 0) {
3787 start_reg = idx_value << 2;
3788 if (command & PACKET3_CP_DMA_CMD_SAIC) {
3790 if (!si_vm_reg_valid(reg)) {
3791 DRM_ERROR("CP DMA Bad SRC register\n");
3795 for (i = 0; i < (command & 0x1fffff); i++) {
3796 reg = start_reg + (4 * i);
3797 if (!si_vm_reg_valid(reg)) {
3798 DRM_ERROR("CP DMA Bad SRC register\n");
3805 if (command & PACKET3_CP_DMA_CMD_DAS) {
3806 /* dst address space is register */
3807 if (((info & 0x00300000) >> 20) == 0) {
3808 start_reg = ib[idx + 2];
3809 if (command & PACKET3_CP_DMA_CMD_DAIC) {
3811 if (!si_vm_reg_valid(reg)) {
3812 DRM_ERROR("CP DMA Bad DST register\n");
3816 for (i = 0; i < (command & 0x1fffff); i++) {
3817 reg = start_reg + (4 * i);
3818 if (!si_vm_reg_valid(reg)) {
3819 DRM_ERROR("CP DMA Bad DST register\n");
3828 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
3834 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
3835 u32 *ib, struct radeon_cs_packet *pkt)
3837 u32 idx = pkt->idx + 1;
3838 u32 idx_value = ib[idx];
3839 u32 start_reg, reg, i;
3841 switch (pkt->opcode) {
3843 case PACKET3_SET_BASE:
3844 case PACKET3_CLEAR_STATE:
3845 case PACKET3_DISPATCH_DIRECT:
3846 case PACKET3_DISPATCH_INDIRECT:
3847 case PACKET3_ALLOC_GDS:
3848 case PACKET3_WRITE_GDS_RAM:
3849 case PACKET3_ATOMIC_GDS:
3850 case PACKET3_ATOMIC:
3851 case PACKET3_OCCLUSION_QUERY:
3852 case PACKET3_SET_PREDICATION:
3853 case PACKET3_COND_EXEC:
3854 case PACKET3_PRED_EXEC:
3855 case PACKET3_CONTEXT_CONTROL:
3856 case PACKET3_STRMOUT_BUFFER_UPDATE:
3857 case PACKET3_WAIT_REG_MEM:
3858 case PACKET3_MEM_WRITE:
3859 case PACKET3_PFP_SYNC_ME:
3860 case PACKET3_SURFACE_SYNC:
3861 case PACKET3_EVENT_WRITE:
3862 case PACKET3_EVENT_WRITE_EOP:
3863 case PACKET3_EVENT_WRITE_EOS:
3864 case PACKET3_SET_CONTEXT_REG:
3865 case PACKET3_SET_CONTEXT_REG_INDIRECT:
3866 case PACKET3_SET_SH_REG:
3867 case PACKET3_SET_SH_REG_OFFSET:
3868 case PACKET3_INCREMENT_DE_COUNTER:
3869 case PACKET3_WAIT_ON_CE_COUNTER:
3870 case PACKET3_WAIT_ON_AVAIL_BUFFER:
3871 case PACKET3_ME_WRITE:
3873 case PACKET3_COPY_DATA:
3874 if ((idx_value & 0xf00) == 0) {
3875 reg = ib[idx + 3] * 4;
3876 if (!si_vm_reg_valid(reg))
3880 case PACKET3_WRITE_DATA:
3881 if ((idx_value & 0xf00) == 0) {
3882 start_reg = ib[idx + 1] * 4;
3883 if (idx_value & 0x10000) {
3884 if (!si_vm_reg_valid(start_reg))
3887 for (i = 0; i < (pkt->count - 2); i++) {
3888 reg = start_reg + (4 * i);
3889 if (!si_vm_reg_valid(reg))
3895 case PACKET3_COND_WRITE:
3896 if (idx_value & 0x100) {
3897 reg = ib[idx + 5] * 4;
3898 if (!si_vm_reg_valid(reg))
3902 case PACKET3_COPY_DW:
3903 if (idx_value & 0x2) {
3904 reg = ib[idx + 3] * 4;
3905 if (!si_vm_reg_valid(reg))
3910 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
3916 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
3920 struct radeon_cs_packet pkt;
3924 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
3925 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
3928 case RADEON_PACKET_TYPE0:
3929 dev_err(rdev->dev, "Packet0 not allowed!\n");
3932 case RADEON_PACKET_TYPE2:
3935 case RADEON_PACKET_TYPE3:
3936 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
3937 if (ib->is_const_ib)
3938 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
3941 case RADEON_RING_TYPE_GFX_INDEX:
3942 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
3944 case CAYMAN_RING_TYPE_CP1_INDEX:
3945 case CAYMAN_RING_TYPE_CP2_INDEX:
3946 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
3949 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
3954 idx += pkt.count + 2;
3957 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
3963 } while (idx < ib->length_dw);
3971 int si_vm_init(struct radeon_device *rdev)
3974 rdev->vm_manager.nvm = 16;
3975 /* base offset of vram pages */
3976 rdev->vm_manager.vram_base_offset = 0;
3981 void si_vm_fini(struct radeon_device *rdev)
3986 * si_vm_set_page - update the page tables using the CP
3988 * @rdev: radeon_device pointer
3989 * @ib: indirect buffer to fill with commands
3990 * @pe: addr of the page entry
3991 * @addr: dst addr to write into pe
3992 * @count: number of page entries to update
3993 * @incr: increase next addr by incr bytes
3994 * @flags: access flags
3996 * Update the page tables using the CP (SI).
3998 void si_vm_set_page(struct radeon_device *rdev,
3999 struct radeon_ib *ib,
4001 uint64_t addr, unsigned count,
4002 uint32_t incr, uint32_t flags)
4004 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4008 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4010 ndw = 2 + count * 2;
4014 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4015 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4016 WRITE_DATA_DST_SEL(1));
4017 ib->ptr[ib->length_dw++] = pe;
4018 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4019 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4020 if (flags & RADEON_VM_PAGE_SYSTEM) {
4021 value = radeon_vm_map_gart(rdev, addr);
4022 value &= 0xFFFFFFFFFFFFF000ULL;
4023 } else if (flags & RADEON_VM_PAGE_VALID) {
4029 value |= r600_flags;
4030 ib->ptr[ib->length_dw++] = value;
4031 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4036 if (flags & RADEON_VM_PAGE_SYSTEM) {
4042 /* for non-physically contiguous pages (system) */
4043 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
4044 ib->ptr[ib->length_dw++] = pe;
4045 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4046 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
4047 if (flags & RADEON_VM_PAGE_SYSTEM) {
4048 value = radeon_vm_map_gart(rdev, addr);
4049 value &= 0xFFFFFFFFFFFFF000ULL;
4050 } else if (flags & RADEON_VM_PAGE_VALID) {
4056 value |= r600_flags;
4057 ib->ptr[ib->length_dw++] = value;
4058 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4067 if (flags & RADEON_VM_PAGE_VALID)
4071 /* for physically contiguous pages (vram) */
4072 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
4073 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4074 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4075 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4076 ib->ptr[ib->length_dw++] = 0;
4077 ib->ptr[ib->length_dw++] = value; /* value */
4078 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4079 ib->ptr[ib->length_dw++] = incr; /* increment size */
4080 ib->ptr[ib->length_dw++] = 0;
4082 addr += (ndw / 2) * incr;
4086 while (ib->length_dw & 0x7)
4087 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
4091 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4093 struct radeon_ring *ring = &rdev->ring[ridx];
4098 /* write new base address */
4099 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4100 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4101 WRITE_DATA_DST_SEL(0)));
4104 radeon_ring_write(ring,
4105 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4107 radeon_ring_write(ring,
4108 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4110 radeon_ring_write(ring, 0);
4111 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4113 /* flush hdp cache */
4114 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4115 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4116 WRITE_DATA_DST_SEL(0)));
4117 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4118 radeon_ring_write(ring, 0);
4119 radeon_ring_write(ring, 0x1);
4121 /* bits 0-15 are the VM contexts0-15 */
4122 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4123 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4124 WRITE_DATA_DST_SEL(0)));
4125 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4126 radeon_ring_write(ring, 0);
4127 radeon_ring_write(ring, 1 << vm->id);
4129 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4130 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4131 radeon_ring_write(ring, 0x0);
4134 void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4136 struct radeon_ring *ring = &rdev->ring[ridx];
4141 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4143 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4145 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4147 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4149 /* flush hdp cache */
4150 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4151 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4152 radeon_ring_write(ring, 1);
4154 /* bits 0-7 are the VM contexts0-7 */
4155 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4156 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4157 radeon_ring_write(ring, 1 << vm->id);
4163 void si_rlc_fini(struct radeon_device *rdev)
4167 /* save restore block */
4168 if (rdev->rlc.save_restore_obj) {
4169 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4170 if (unlikely(r != 0))
4171 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4172 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4173 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4175 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4176 rdev->rlc.save_restore_obj = NULL;
4179 /* clear state block */
4180 if (rdev->rlc.clear_state_obj) {
4181 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4182 if (unlikely(r != 0))
4183 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4184 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4185 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4187 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4188 rdev->rlc.clear_state_obj = NULL;
4192 int si_rlc_init(struct radeon_device *rdev)
4196 /* save restore block */
4197 if (rdev->rlc.save_restore_obj == NULL) {
4198 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
4199 RADEON_GEM_DOMAIN_VRAM, NULL,
4200 &rdev->rlc.save_restore_obj);
4202 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4207 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4208 if (unlikely(r != 0)) {
4212 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4213 &rdev->rlc.save_restore_gpu_addr);
4214 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4216 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4221 /* clear state block */
4222 if (rdev->rlc.clear_state_obj == NULL) {
4223 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
4224 RADEON_GEM_DOMAIN_VRAM, NULL,
4225 &rdev->rlc.clear_state_obj);
4227 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4232 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4233 if (unlikely(r != 0)) {
4237 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4238 &rdev->rlc.clear_state_gpu_addr);
4239 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4241 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4249 static void si_rlc_stop(struct radeon_device *rdev)
4251 WREG32(RLC_CNTL, 0);
4254 static void si_rlc_start(struct radeon_device *rdev)
4256 WREG32(RLC_CNTL, RLC_ENABLE);
4259 static int si_rlc_resume(struct radeon_device *rdev)
4262 const __be32 *fw_data;
4269 WREG32(RLC_RL_BASE, 0);
4270 WREG32(RLC_RL_SIZE, 0);
4271 WREG32(RLC_LB_CNTL, 0);
4272 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
4273 WREG32(RLC_LB_CNTR_INIT, 0);
4275 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4276 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4278 WREG32(RLC_MC_CNTL, 0);
4279 WREG32(RLC_UCODE_CNTL, 0);
4281 fw_data = (const __be32 *)rdev->rlc_fw->data;
4282 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
4283 WREG32(RLC_UCODE_ADDR, i);
4284 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4286 WREG32(RLC_UCODE_ADDR, 0);
4293 static void si_enable_interrupts(struct radeon_device *rdev)
4295 u32 ih_cntl = RREG32(IH_CNTL);
4296 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4298 ih_cntl |= ENABLE_INTR;
4299 ih_rb_cntl |= IH_RB_ENABLE;
4300 WREG32(IH_CNTL, ih_cntl);
4301 WREG32(IH_RB_CNTL, ih_rb_cntl);
4302 rdev->ih.enabled = true;
4305 static void si_disable_interrupts(struct radeon_device *rdev)
4307 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4308 u32 ih_cntl = RREG32(IH_CNTL);
4310 ih_rb_cntl &= ~IH_RB_ENABLE;
4311 ih_cntl &= ~ENABLE_INTR;
4312 WREG32(IH_RB_CNTL, ih_rb_cntl);
4313 WREG32(IH_CNTL, ih_cntl);
4314 /* set rptr, wptr to 0 */
4315 WREG32(IH_RB_RPTR, 0);
4316 WREG32(IH_RB_WPTR, 0);
4317 rdev->ih.enabled = false;
4321 static void si_disable_interrupt_state(struct radeon_device *rdev)
4325 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4326 WREG32(CP_INT_CNTL_RING1, 0);
4327 WREG32(CP_INT_CNTL_RING2, 0);
4328 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4329 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
4330 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4331 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
4332 WREG32(GRBM_INT_CNTL, 0);
4333 if (rdev->num_crtc >= 2) {
4334 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4335 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4337 if (rdev->num_crtc >= 4) {
4338 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4339 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4341 if (rdev->num_crtc >= 6) {
4342 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4343 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4346 if (rdev->num_crtc >= 2) {
4347 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4348 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4350 if (rdev->num_crtc >= 4) {
4351 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4352 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4354 if (rdev->num_crtc >= 6) {
4355 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4356 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4359 if (!ASIC_IS_NODCE(rdev)) {
4360 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4362 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4363 WREG32(DC_HPD1_INT_CONTROL, tmp);
4364 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4365 WREG32(DC_HPD2_INT_CONTROL, tmp);
4366 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4367 WREG32(DC_HPD3_INT_CONTROL, tmp);
4368 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4369 WREG32(DC_HPD4_INT_CONTROL, tmp);
4370 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4371 WREG32(DC_HPD5_INT_CONTROL, tmp);
4372 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4373 WREG32(DC_HPD6_INT_CONTROL, tmp);
4377 static int si_irq_init(struct radeon_device *rdev)
4381 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
4384 ret = r600_ih_ring_alloc(rdev);
4389 si_disable_interrupts(rdev);
4392 ret = si_rlc_resume(rdev);
4394 r600_ih_ring_fini(rdev);
4398 /* setup interrupt control */
4399 /* set dummy read address to ring address */
4400 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
4401 interrupt_cntl = RREG32(INTERRUPT_CNTL);
4402 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
4403 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
4405 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
4406 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
4407 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
4408 WREG32(INTERRUPT_CNTL, interrupt_cntl);
4410 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
4411 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
4413 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
4414 IH_WPTR_OVERFLOW_CLEAR |
4417 if (rdev->wb.enabled)
4418 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
4420 /* set the writeback address whether it's enabled or not */
4421 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
4422 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
4424 WREG32(IH_RB_CNTL, ih_rb_cntl);
4426 /* set rptr, wptr to 0 */
4427 WREG32(IH_RB_RPTR, 0);
4428 WREG32(IH_RB_WPTR, 0);
4430 /* Default settings for IH_CNTL (disabled at first) */
4431 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
4432 /* RPTR_REARM only works if msi's are enabled */
4433 if (rdev->msi_enabled)
4434 ih_cntl |= RPTR_REARM;
4435 WREG32(IH_CNTL, ih_cntl);
4437 /* force the active interrupt state to all disabled */
4438 si_disable_interrupt_state(rdev);
4440 pci_set_master(rdev->pdev);
4443 si_enable_interrupts(rdev);
4448 int si_irq_set(struct radeon_device *rdev)
4450 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4451 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4452 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4453 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
4454 u32 grbm_int_cntl = 0;
4455 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
4456 u32 dma_cntl, dma_cntl1;
4458 if (!rdev->irq.installed) {
4459 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4462 /* don't enable anything if the ih is disabled */
4463 if (!rdev->ih.enabled) {
4464 si_disable_interrupts(rdev);
4465 /* force the active interrupt state to all disabled */
4466 si_disable_interrupt_state(rdev);
4470 if (!ASIC_IS_NODCE(rdev)) {
4471 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4472 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4473 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4474 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4475 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4476 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4479 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4480 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4482 /* enable CP interrupts on all rings */
4483 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4484 DRM_DEBUG("si_irq_set: sw int gfx\n");
4485 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4487 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4488 DRM_DEBUG("si_irq_set: sw int cp1\n");
4489 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4491 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4492 DRM_DEBUG("si_irq_set: sw int cp2\n");
4493 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4495 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4496 DRM_DEBUG("si_irq_set: sw int dma\n");
4497 dma_cntl |= TRAP_ENABLE;
4500 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4501 DRM_DEBUG("si_irq_set: sw int dma1\n");
4502 dma_cntl1 |= TRAP_ENABLE;
4504 if (rdev->irq.crtc_vblank_int[0] ||
4505 atomic_read(&rdev->irq.pflip[0])) {
4506 DRM_DEBUG("si_irq_set: vblank 0\n");
4507 crtc1 |= VBLANK_INT_MASK;
4509 if (rdev->irq.crtc_vblank_int[1] ||
4510 atomic_read(&rdev->irq.pflip[1])) {
4511 DRM_DEBUG("si_irq_set: vblank 1\n");
4512 crtc2 |= VBLANK_INT_MASK;
4514 if (rdev->irq.crtc_vblank_int[2] ||
4515 atomic_read(&rdev->irq.pflip[2])) {
4516 DRM_DEBUG("si_irq_set: vblank 2\n");
4517 crtc3 |= VBLANK_INT_MASK;
4519 if (rdev->irq.crtc_vblank_int[3] ||
4520 atomic_read(&rdev->irq.pflip[3])) {
4521 DRM_DEBUG("si_irq_set: vblank 3\n");
4522 crtc4 |= VBLANK_INT_MASK;
4524 if (rdev->irq.crtc_vblank_int[4] ||
4525 atomic_read(&rdev->irq.pflip[4])) {
4526 DRM_DEBUG("si_irq_set: vblank 4\n");
4527 crtc5 |= VBLANK_INT_MASK;
4529 if (rdev->irq.crtc_vblank_int[5] ||
4530 atomic_read(&rdev->irq.pflip[5])) {
4531 DRM_DEBUG("si_irq_set: vblank 5\n");
4532 crtc6 |= VBLANK_INT_MASK;
4534 if (rdev->irq.hpd[0]) {
4535 DRM_DEBUG("si_irq_set: hpd 1\n");
4536 hpd1 |= DC_HPDx_INT_EN;
4538 if (rdev->irq.hpd[1]) {
4539 DRM_DEBUG("si_irq_set: hpd 2\n");
4540 hpd2 |= DC_HPDx_INT_EN;
4542 if (rdev->irq.hpd[2]) {
4543 DRM_DEBUG("si_irq_set: hpd 3\n");
4544 hpd3 |= DC_HPDx_INT_EN;
4546 if (rdev->irq.hpd[3]) {
4547 DRM_DEBUG("si_irq_set: hpd 4\n");
4548 hpd4 |= DC_HPDx_INT_EN;
4550 if (rdev->irq.hpd[4]) {
4551 DRM_DEBUG("si_irq_set: hpd 5\n");
4552 hpd5 |= DC_HPDx_INT_EN;
4554 if (rdev->irq.hpd[5]) {
4555 DRM_DEBUG("si_irq_set: hpd 6\n");
4556 hpd6 |= DC_HPDx_INT_EN;
4559 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
4560 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
4561 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
4563 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
4564 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
4566 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4568 if (rdev->num_crtc >= 2) {
4569 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4570 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4572 if (rdev->num_crtc >= 4) {
4573 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4574 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4576 if (rdev->num_crtc >= 6) {
4577 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4578 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4581 if (rdev->num_crtc >= 2) {
4582 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4583 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4585 if (rdev->num_crtc >= 4) {
4586 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4587 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4589 if (rdev->num_crtc >= 6) {
4590 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4591 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4594 if (!ASIC_IS_NODCE(rdev)) {
4595 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4596 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4597 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4598 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4599 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4600 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4606 static inline void si_irq_ack(struct radeon_device *rdev)
4610 if (ASIC_IS_NODCE(rdev))
4613 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4614 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4615 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4616 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4617 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4618 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4619 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4620 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4621 if (rdev->num_crtc >= 4) {
4622 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4623 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4625 if (rdev->num_crtc >= 6) {
4626 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4627 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4630 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4631 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4632 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4633 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4634 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4635 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4636 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4637 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4638 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4639 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4640 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4641 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4643 if (rdev->num_crtc >= 4) {
4644 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4645 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4646 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4647 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4648 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4649 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4650 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4651 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4652 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4653 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4654 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4655 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4658 if (rdev->num_crtc >= 6) {
4659 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4660 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4661 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4662 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4663 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4664 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4665 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4666 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4667 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4668 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4669 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4670 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4673 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4674 tmp = RREG32(DC_HPD1_INT_CONTROL);
4675 tmp |= DC_HPDx_INT_ACK;
4676 WREG32(DC_HPD1_INT_CONTROL, tmp);
4678 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4679 tmp = RREG32(DC_HPD2_INT_CONTROL);
4680 tmp |= DC_HPDx_INT_ACK;
4681 WREG32(DC_HPD2_INT_CONTROL, tmp);
4683 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4684 tmp = RREG32(DC_HPD3_INT_CONTROL);
4685 tmp |= DC_HPDx_INT_ACK;
4686 WREG32(DC_HPD3_INT_CONTROL, tmp);
4688 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4689 tmp = RREG32(DC_HPD4_INT_CONTROL);
4690 tmp |= DC_HPDx_INT_ACK;
4691 WREG32(DC_HPD4_INT_CONTROL, tmp);
4693 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4694 tmp = RREG32(DC_HPD5_INT_CONTROL);
4695 tmp |= DC_HPDx_INT_ACK;
4696 WREG32(DC_HPD5_INT_CONTROL, tmp);
4698 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4699 tmp = RREG32(DC_HPD5_INT_CONTROL);
4700 tmp |= DC_HPDx_INT_ACK;
4701 WREG32(DC_HPD6_INT_CONTROL, tmp);
4705 static void si_irq_disable(struct radeon_device *rdev)
4707 si_disable_interrupts(rdev);
4708 /* Wait and acknowledge irq */
4711 si_disable_interrupt_state(rdev);
4714 static void si_irq_suspend(struct radeon_device *rdev)
4716 si_irq_disable(rdev);
4720 static void si_irq_fini(struct radeon_device *rdev)
4722 si_irq_suspend(rdev);
4723 r600_ih_ring_fini(rdev);
4726 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
4730 if (rdev->wb.enabled)
4731 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4733 wptr = RREG32(IH_RB_WPTR);
4735 if (wptr & RB_OVERFLOW) {
4736 /* When a ring buffer overflow happen start parsing interrupt
4737 * from the last not overwritten vector (wptr + 16). Hopefully
4738 * this should allow us to catchup.
4740 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4741 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4742 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4743 tmp = RREG32(IH_RB_CNTL);
4744 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4745 WREG32(IH_RB_CNTL, tmp);
4747 return (wptr & rdev->ih.ptr_mask);
4751 * Each IV ring entry is 128 bits:
4752 * [7:0] - interrupt source id
4754 * [59:32] - interrupt source data
4755 * [63:60] - reserved
4758 * [127:80] - reserved
4760 int si_irq_process(struct radeon_device *rdev)
4764 u32 src_id, src_data, ring_id;
4766 bool queue_hotplug = false;
4768 if (!rdev->ih.enabled || rdev->shutdown)
4771 wptr = si_get_ih_wptr(rdev);
4774 /* is somebody else already processing irqs? */
4775 if (atomic_xchg(&rdev->ih.lock, 1))
4778 rptr = rdev->ih.rptr;
4779 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4781 /* Order reading of wptr vs. reading of IH ring data */
4784 /* display interrupts */
4787 while (rptr != wptr) {
4788 /* wptr/rptr are in bytes! */
4789 ring_index = rptr / 4;
4790 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4791 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4792 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
4795 case 1: /* D1 vblank/vline */
4797 case 0: /* D1 vblank */
4798 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4799 if (rdev->irq.crtc_vblank_int[0]) {
4800 drm_handle_vblank(rdev->ddev, 0);
4801 rdev->pm.vblank_sync = true;
4802 wake_up(&rdev->irq.vblank_queue);
4804 if (atomic_read(&rdev->irq.pflip[0]))
4805 radeon_crtc_handle_flip(rdev, 0);
4806 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4807 DRM_DEBUG("IH: D1 vblank\n");
4810 case 1: /* D1 vline */
4811 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4812 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4813 DRM_DEBUG("IH: D1 vline\n");
4817 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4821 case 2: /* D2 vblank/vline */
4823 case 0: /* D2 vblank */
4824 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4825 if (rdev->irq.crtc_vblank_int[1]) {
4826 drm_handle_vblank(rdev->ddev, 1);
4827 rdev->pm.vblank_sync = true;
4828 wake_up(&rdev->irq.vblank_queue);
4830 if (atomic_read(&rdev->irq.pflip[1]))
4831 radeon_crtc_handle_flip(rdev, 1);
4832 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4833 DRM_DEBUG("IH: D2 vblank\n");
4836 case 1: /* D2 vline */
4837 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4838 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4839 DRM_DEBUG("IH: D2 vline\n");
4843 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4847 case 3: /* D3 vblank/vline */
4849 case 0: /* D3 vblank */
4850 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4851 if (rdev->irq.crtc_vblank_int[2]) {
4852 drm_handle_vblank(rdev->ddev, 2);
4853 rdev->pm.vblank_sync = true;
4854 wake_up(&rdev->irq.vblank_queue);
4856 if (atomic_read(&rdev->irq.pflip[2]))
4857 radeon_crtc_handle_flip(rdev, 2);
4858 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4859 DRM_DEBUG("IH: D3 vblank\n");
4862 case 1: /* D3 vline */
4863 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4864 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4865 DRM_DEBUG("IH: D3 vline\n");
4869 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4873 case 4: /* D4 vblank/vline */
4875 case 0: /* D4 vblank */
4876 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4877 if (rdev->irq.crtc_vblank_int[3]) {
4878 drm_handle_vblank(rdev->ddev, 3);
4879 rdev->pm.vblank_sync = true;
4880 wake_up(&rdev->irq.vblank_queue);
4882 if (atomic_read(&rdev->irq.pflip[3]))
4883 radeon_crtc_handle_flip(rdev, 3);
4884 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4885 DRM_DEBUG("IH: D4 vblank\n");
4888 case 1: /* D4 vline */
4889 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4890 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4891 DRM_DEBUG("IH: D4 vline\n");
4895 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4899 case 5: /* D5 vblank/vline */
4901 case 0: /* D5 vblank */
4902 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4903 if (rdev->irq.crtc_vblank_int[4]) {
4904 drm_handle_vblank(rdev->ddev, 4);
4905 rdev->pm.vblank_sync = true;
4906 wake_up(&rdev->irq.vblank_queue);
4908 if (atomic_read(&rdev->irq.pflip[4]))
4909 radeon_crtc_handle_flip(rdev, 4);
4910 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4911 DRM_DEBUG("IH: D5 vblank\n");
4914 case 1: /* D5 vline */
4915 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4916 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4917 DRM_DEBUG("IH: D5 vline\n");
4921 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4925 case 6: /* D6 vblank/vline */
4927 case 0: /* D6 vblank */
4928 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4929 if (rdev->irq.crtc_vblank_int[5]) {
4930 drm_handle_vblank(rdev->ddev, 5);
4931 rdev->pm.vblank_sync = true;
4932 wake_up(&rdev->irq.vblank_queue);
4934 if (atomic_read(&rdev->irq.pflip[5]))
4935 radeon_crtc_handle_flip(rdev, 5);
4936 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4937 DRM_DEBUG("IH: D6 vblank\n");
4940 case 1: /* D6 vline */
4941 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4942 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4943 DRM_DEBUG("IH: D6 vline\n");
4947 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4951 case 42: /* HPD hotplug */
4954 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4955 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4956 queue_hotplug = true;
4957 DRM_DEBUG("IH: HPD1\n");
4961 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4962 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4963 queue_hotplug = true;
4964 DRM_DEBUG("IH: HPD2\n");
4968 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4969 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4970 queue_hotplug = true;
4971 DRM_DEBUG("IH: HPD3\n");
4975 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4976 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4977 queue_hotplug = true;
4978 DRM_DEBUG("IH: HPD4\n");
4982 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4983 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4984 queue_hotplug = true;
4985 DRM_DEBUG("IH: HPD5\n");
4989 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4990 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4991 queue_hotplug = true;
4992 DRM_DEBUG("IH: HPD6\n");
4996 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5002 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5003 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5004 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5005 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5006 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5007 /* reset addr and status */
5008 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5010 case 176: /* RINGID0 CP_INT */
5011 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5013 case 177: /* RINGID1 CP_INT */
5014 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5016 case 178: /* RINGID2 CP_INT */
5017 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5019 case 181: /* CP EOP event */
5020 DRM_DEBUG("IH: CP EOP\n");
5023 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5026 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5029 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5033 case 224: /* DMA trap event */
5034 DRM_DEBUG("IH: DMA trap\n");
5035 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5037 case 233: /* GUI IDLE */
5038 DRM_DEBUG("IH: GUI idle\n");
5040 case 244: /* DMA trap event */
5041 DRM_DEBUG("IH: DMA1 trap\n");
5042 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5045 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5049 /* wptr/rptr are in bytes! */
5051 rptr &= rdev->ih.ptr_mask;
5054 schedule_work(&rdev->hotplug_work);
5055 rdev->ih.rptr = rptr;
5056 WREG32(IH_RB_RPTR, rdev->ih.rptr);
5057 atomic_set(&rdev->ih.lock, 0);
5059 /* make sure wptr hasn't changed while processing */
5060 wptr = si_get_ih_wptr(rdev);
5068 * si_copy_dma - copy pages using the DMA engine
5070 * @rdev: radeon_device pointer
5071 * @src_offset: src GPU address
5072 * @dst_offset: dst GPU address
5073 * @num_gpu_pages: number of GPU pages to xfer
5074 * @fence: radeon fence object
5076 * Copy GPU paging using the DMA engine (SI).
5077 * Used by the radeon ttm implementation to move pages if
5078 * registered as the asic copy callback.
5080 int si_copy_dma(struct radeon_device *rdev,
5081 uint64_t src_offset, uint64_t dst_offset,
5082 unsigned num_gpu_pages,
5083 struct radeon_fence **fence)
5085 struct radeon_semaphore *sem = NULL;
5086 int ring_index = rdev->asic->copy.dma_ring_index;
5087 struct radeon_ring *ring = &rdev->ring[ring_index];
5088 u32 size_in_bytes, cur_size_in_bytes;
5092 r = radeon_semaphore_create(rdev, &sem);
5094 DRM_ERROR("radeon: moving bo (%d).\n", r);
5098 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
5099 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
5100 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5102 DRM_ERROR("radeon: moving bo (%d).\n", r);
5103 radeon_semaphore_free(rdev, &sem, NULL);
5107 if (radeon_fence_need_sync(*fence, ring->idx)) {
5108 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5110 radeon_fence_note_sync(*fence, ring->idx);
5112 radeon_semaphore_free(rdev, &sem, NULL);
5115 for (i = 0; i < num_loops; i++) {
5116 cur_size_in_bytes = size_in_bytes;
5117 if (cur_size_in_bytes > 0xFFFFF)
5118 cur_size_in_bytes = 0xFFFFF;
5119 size_in_bytes -= cur_size_in_bytes;
5120 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
5121 radeon_ring_write(ring, dst_offset & 0xffffffff);
5122 radeon_ring_write(ring, src_offset & 0xffffffff);
5123 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5124 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5125 src_offset += cur_size_in_bytes;
5126 dst_offset += cur_size_in_bytes;
5129 r = radeon_fence_emit(rdev, fence, ring->idx);
5131 radeon_ring_unlock_undo(rdev, ring);
5135 radeon_ring_unlock_commit(rdev, ring);
5136 radeon_semaphore_free(rdev, &sem, *fence);
5142 * startup/shutdown callbacks
5144 static int si_startup(struct radeon_device *rdev)
5146 struct radeon_ring *ring;
5149 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5150 !rdev->rlc_fw || !rdev->mc_fw) {
5151 r = si_init_microcode(rdev);
5153 DRM_ERROR("Failed to load firmware!\n");
5158 r = si_mc_load_microcode(rdev);
5160 DRM_ERROR("Failed to load MC firmware!\n");
5164 r = r600_vram_scratch_init(rdev);
5168 si_mc_program(rdev);
5169 r = si_pcie_gart_enable(rdev);
5174 /* allocate rlc buffers */
5175 r = si_rlc_init(rdev);
5177 DRM_ERROR("Failed to init rlc BOs!\n");
5181 /* allocate wb buffer */
5182 r = radeon_wb_init(rdev);
5186 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5188 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5192 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5194 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5198 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5200 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5204 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5206 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5210 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5212 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5216 r = rv770_uvd_resume(rdev);
5218 r = radeon_fence_driver_start_ring(rdev,
5219 R600_RING_TYPE_UVD_INDEX);
5221 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5224 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5227 r = si_irq_init(rdev);
5229 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5230 radeon_irq_kms_fini(rdev);
5235 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5236 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5237 CP_RB0_RPTR, CP_RB0_WPTR,
5238 0, 0xfffff, RADEON_CP_PACKET2);
5242 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5243 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
5244 CP_RB1_RPTR, CP_RB1_WPTR,
5245 0, 0xfffff, RADEON_CP_PACKET2);
5249 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5250 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
5251 CP_RB2_RPTR, CP_RB2_WPTR,
5252 0, 0xfffff, RADEON_CP_PACKET2);
5256 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5257 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5258 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
5259 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
5260 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5264 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5265 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
5266 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
5267 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
5268 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5272 r = si_cp_load_microcode(rdev);
5275 r = si_cp_resume(rdev);
5279 r = cayman_dma_resume(rdev);
5283 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5284 if (ring->ring_size) {
5285 r = radeon_ring_init(rdev, ring, ring->ring_size,
5286 R600_WB_UVD_RPTR_OFFSET,
5287 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5288 0, 0xfffff, RADEON_CP_PACKET2);
5290 r = r600_uvd_init(rdev);
5292 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
5295 r = radeon_ib_pool_init(rdev);
5297 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5301 r = radeon_vm_manager_init(rdev);
5303 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
5310 int si_resume(struct radeon_device *rdev)
5314 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5315 * posting will perform necessary task to bring back GPU into good
5319 atom_asic_init(rdev->mode_info.atom_context);
5321 /* init golden registers */
5322 si_init_golden_registers(rdev);
5324 rdev->accel_working = true;
5325 r = si_startup(rdev);
5327 DRM_ERROR("si startup failed on resume\n");
5328 rdev->accel_working = false;
5336 int si_suspend(struct radeon_device *rdev)
5338 radeon_vm_manager_fini(rdev);
5339 si_cp_enable(rdev, false);
5340 cayman_dma_stop(rdev);
5341 r600_uvd_rbc_stop(rdev);
5342 radeon_uvd_suspend(rdev);
5343 si_irq_suspend(rdev);
5344 radeon_wb_disable(rdev);
5345 si_pcie_gart_disable(rdev);
5349 /* Plan is to move initialization in that function and use
5350 * helper function so that radeon_device_init pretty much
5351 * do nothing more than calling asic specific function. This
5352 * should also allow to remove a bunch of callback function
5355 int si_init(struct radeon_device *rdev)
5357 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5361 if (!radeon_get_bios(rdev)) {
5362 if (ASIC_IS_AVIVO(rdev))
5365 /* Must be an ATOMBIOS */
5366 if (!rdev->is_atom_bios) {
5367 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
5370 r = radeon_atombios_init(rdev);
5374 /* Post card if necessary */
5375 if (!radeon_card_posted(rdev)) {
5377 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5380 DRM_INFO("GPU not posted. posting now...\n");
5381 atom_asic_init(rdev->mode_info.atom_context);
5383 /* init golden registers */
5384 si_init_golden_registers(rdev);
5385 /* Initialize scratch registers */
5386 si_scratch_init(rdev);
5387 /* Initialize surface registers */
5388 radeon_surface_init(rdev);
5389 /* Initialize clocks */
5390 radeon_get_clock_info(rdev->ddev);
5393 r = radeon_fence_driver_init(rdev);
5397 /* initialize memory controller */
5398 r = si_mc_init(rdev);
5401 /* Memory manager */
5402 r = radeon_bo_init(rdev);
5406 r = radeon_irq_kms_init(rdev);
5410 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5411 ring->ring_obj = NULL;
5412 r600_ring_init(rdev, ring, 1024 * 1024);
5414 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5415 ring->ring_obj = NULL;
5416 r600_ring_init(rdev, ring, 1024 * 1024);
5418 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5419 ring->ring_obj = NULL;
5420 r600_ring_init(rdev, ring, 1024 * 1024);
5422 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5423 ring->ring_obj = NULL;
5424 r600_ring_init(rdev, ring, 64 * 1024);
5426 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5427 ring->ring_obj = NULL;
5428 r600_ring_init(rdev, ring, 64 * 1024);
5430 r = radeon_uvd_init(rdev);
5432 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5433 ring->ring_obj = NULL;
5434 r600_ring_init(rdev, ring, 4096);
5437 rdev->ih.ring_obj = NULL;
5438 r600_ih_ring_init(rdev, 64 * 1024);
5440 r = r600_pcie_gart_init(rdev);
5444 rdev->accel_working = true;
5445 r = si_startup(rdev);
5447 dev_err(rdev->dev, "disabling GPU acceleration\n");
5449 cayman_dma_fini(rdev);
5452 radeon_wb_fini(rdev);
5453 radeon_ib_pool_fini(rdev);
5454 radeon_vm_manager_fini(rdev);
5455 radeon_irq_kms_fini(rdev);
5456 si_pcie_gart_fini(rdev);
5457 rdev->accel_working = false;
5460 /* Don't start up if the MC ucode is missing.
5461 * The default clocks and voltages before the MC ucode
5462 * is loaded are not suffient for advanced operations.
5465 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5472 void si_fini(struct radeon_device *rdev)
5475 cayman_dma_fini(rdev);
5478 radeon_wb_fini(rdev);
5479 radeon_vm_manager_fini(rdev);
5480 radeon_ib_pool_fini(rdev);
5481 radeon_irq_kms_fini(rdev);
5482 radeon_uvd_fini(rdev);
5483 si_pcie_gart_fini(rdev);
5484 r600_vram_scratch_fini(rdev);
5485 radeon_gem_fini(rdev);
5486 radeon_fence_driver_fini(rdev);
5487 radeon_bo_fini(rdev);
5488 radeon_atombios_fini(rdev);
5494 * si_get_gpu_clock_counter - return GPU clock counter snapshot
5496 * @rdev: radeon_device pointer
5498 * Fetches a GPU clock counter snapshot (SI).
5499 * Returns the 64 bit clock counter snapshot.
5501 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
5505 mutex_lock(&rdev->gpu_clock_mutex);
5506 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5507 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
5508 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5509 mutex_unlock(&rdev->gpu_clock_mutex);
5513 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
5515 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
5518 /* bypass vclk and dclk with bclk */
5519 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5520 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
5521 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5523 /* put PLL in bypass mode */
5524 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
5526 if (!vclk || !dclk) {
5527 /* keep the Bypass mode, put PLL to sleep */
5528 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5532 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
5533 16384, 0x03FFFFFF, 0, 128, 5,
5534 &fb_div, &vclk_div, &dclk_div);
5538 /* set RESET_ANTI_MUX to 0 */
5539 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
5541 /* set VCO_MODE to 1 */
5542 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
5544 /* toggle UPLL_SLEEP to 1 then back to 0 */
5545 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5546 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
5548 /* deassert UPLL_RESET */
5549 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5553 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
5557 /* assert UPLL_RESET again */
5558 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
5560 /* disable spread spectrum. */
5561 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
5563 /* set feedback divider */
5564 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
5566 /* set ref divider to 0 */
5567 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
5569 if (fb_div < 307200)
5570 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
5572 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
5574 /* set PDIV_A and PDIV_B */
5575 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5576 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
5577 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
5579 /* give the PLL some time to settle */
5582 /* deassert PLL_RESET */
5583 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5587 /* switch from bypass mode to normal mode */
5588 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
5590 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
5594 /* switch VCLK and DCLK selection */
5595 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5596 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
5597 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));