projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: fill in GPU init for Hainan (v2)
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c
AD		 24#include <linux/firmware.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include <linux/module.h>
760285e7 28#include <drm/drmP.h>
43b3cd99
AD		 29#include "radeon.h"
30#include "radeon_asic.h"
760285e7 31#include <drm/radeon_drm.h>
43b3cd99
AD		 32#include "sid.h"
33#include "atom.h"
48c0c902 34#include "si_blit_shaders.h"
43b3cd99 35
0f0de06c
AD		 36#define SI_PFP_UCODE_SIZE 2144
37#define SI_PM4_UCODE_SIZE 2144
38#define SI_CE_UCODE_SIZE 2144
39#define SI_RLC_UCODE_SIZE 2048
40#define SI_MC_UCODE_SIZE 7769
bcc7f5d2 41#define OLAND_MC_UCODE_SIZE 7863
0f0de06c
AD		 42
43MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
44MODULE_FIRMWARE("radeon/TAHITI_me.bin");
45MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
46MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
47MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
49MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
50MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
51MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
52MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
53MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
54MODULE_FIRMWARE("radeon/VERDE_me.bin");
55MODULE_FIRMWARE("radeon/VERDE_ce.bin");
56MODULE_FIRMWARE("radeon/VERDE_mc.bin");
57MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
bcc7f5d2
AD		 58MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
59MODULE_FIRMWARE("radeon/OLAND_me.bin");
60MODULE_FIRMWARE("radeon/OLAND_ce.bin");
61MODULE_FIRMWARE("radeon/OLAND_mc.bin");
62MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
0f0de06c 63
25a857fb
AD		 64extern int r600_ih_ring_alloc(struct radeon_device *rdev);
65extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 66extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 67extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 69extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 70extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 71extern bool evergreen_is_display_hung(struct radeon_device *rdev);
0a96d72b 72
205996c0
AD		 73static const u32 tahiti_golden_rlc_registers[] =
74{
75 0xc424, 0xffffffff, 0x00601005,
76 0xc47c, 0xffffffff, 0x10104040,
77 0xc488, 0xffffffff, 0x0100000a,
78 0xc314, 0xffffffff, 0x00000800,
79 0xc30c, 0xffffffff, 0x800000f4,
80 0xf4a8, 0xffffffff, 0x00000000
81};
82
83static const u32 tahiti_golden_registers[] =
84{
85 0x9a10, 0x00010000, 0x00018208,
86 0x9830, 0xffffffff, 0x00000000,
87 0x9834, 0xf00fffff, 0x00000400,
88 0x9838, 0x0002021c, 0x00020200,
89 0xc78, 0x00000080, 0x00000000,
90 0xd030, 0x000300c0, 0x00800040,
91 0xd830, 0x000300c0, 0x00800040,
92 0x5bb0, 0x000000f0, 0x00000070,
93 0x5bc0, 0x00200000, 0x50100000,
94 0x7030, 0x31000311, 0x00000011,
95 0x277c, 0x00000003, 0x000007ff,
96 0x240c, 0x000007ff, 0x00000000,
97 0x8a14, 0xf000001f, 0x00000007,
98 0x8b24, 0xffffffff, 0x00ffffff,
99 0x8b10, 0x0000ff0f, 0x00000000,
100 0x28a4c, 0x07ffffff, 0x4e000000,
101 0x28350, 0x3f3f3fff, 0x2a00126a,
102 0x30, 0x000000ff, 0x0040,
103 0x34, 0x00000040, 0x00004040,
104 0x9100, 0x07ffffff, 0x03000000,
105 0x8e88, 0x01ff1f3f, 0x00000000,
106 0x8e84, 0x01ff1f3f, 0x00000000,
107 0x9060, 0x0000007f, 0x00000020,
108 0x9508, 0x00010000, 0x00010000,
109 0xac14, 0x00000200, 0x000002fb,
110 0xac10, 0xffffffff, 0x0000543b,
111 0xac0c, 0xffffffff, 0xa9210876,
112 0x88d0, 0xffffffff, 0x000fff40,
113 0x88d4, 0x0000001f, 0x00000010,
114 0x1410, 0x20000000, 0x20fffed8,
115 0x15c0, 0x000c0fc0, 0x000c0400
116};
117
118static const u32 tahiti_golden_registers2[] =
119{
120 0xc64, 0x00000001, 0x00000001
121};
122
123static const u32 pitcairn_golden_rlc_registers[] =
124{
125 0xc424, 0xffffffff, 0x00601004,
126 0xc47c, 0xffffffff, 0x10102020,
127 0xc488, 0xffffffff, 0x01000020,
128 0xc314, 0xffffffff, 0x00000800,
129 0xc30c, 0xffffffff, 0x800000a4
130};
131
132static const u32 pitcairn_golden_registers[] =
133{
134 0x9a10, 0x00010000, 0x00018208,
135 0x9830, 0xffffffff, 0x00000000,
136 0x9834, 0xf00fffff, 0x00000400,
137 0x9838, 0x0002021c, 0x00020200,
138 0xc78, 0x00000080, 0x00000000,
139 0xd030, 0x000300c0, 0x00800040,
140 0xd830, 0x000300c0, 0x00800040,
141 0x5bb0, 0x000000f0, 0x00000070,
142 0x5bc0, 0x00200000, 0x50100000,
143 0x7030, 0x31000311, 0x00000011,
144 0x2ae4, 0x00073ffe, 0x000022a2,
145 0x240c, 0x000007ff, 0x00000000,
146 0x8a14, 0xf000001f, 0x00000007,
147 0x8b24, 0xffffffff, 0x00ffffff,
148 0x8b10, 0x0000ff0f, 0x00000000,
149 0x28a4c, 0x07ffffff, 0x4e000000,
150 0x28350, 0x3f3f3fff, 0x2a00126a,
151 0x30, 0x000000ff, 0x0040,
152 0x34, 0x00000040, 0x00004040,
153 0x9100, 0x07ffffff, 0x03000000,
154 0x9060, 0x0000007f, 0x00000020,
155 0x9508, 0x00010000, 0x00010000,
156 0xac14, 0x000003ff, 0x000000f7,
157 0xac10, 0xffffffff, 0x00000000,
158 0xac0c, 0xffffffff, 0x32761054,
159 0x88d4, 0x0000001f, 0x00000010,
160 0x15c0, 0x000c0fc0, 0x000c0400
161};
162
163static const u32 verde_golden_rlc_registers[] =
164{
165 0xc424, 0xffffffff, 0x033f1005,
166 0xc47c, 0xffffffff, 0x10808020,
167 0xc488, 0xffffffff, 0x00800008,
168 0xc314, 0xffffffff, 0x00001000,
169 0xc30c, 0xffffffff, 0x80010014
170};
171
172static const u32 verde_golden_registers[] =
173{
174 0x9a10, 0x00010000, 0x00018208,
175 0x9830, 0xffffffff, 0x00000000,
176 0x9834, 0xf00fffff, 0x00000400,
177 0x9838, 0x0002021c, 0x00020200,
178 0xc78, 0x00000080, 0x00000000,
179 0xd030, 0x000300c0, 0x00800040,
180 0xd030, 0x000300c0, 0x00800040,
181 0xd830, 0x000300c0, 0x00800040,
182 0xd830, 0x000300c0, 0x00800040,
183 0x5bb0, 0x000000f0, 0x00000070,
184 0x5bc0, 0x00200000, 0x50100000,
185 0x7030, 0x31000311, 0x00000011,
186 0x2ae4, 0x00073ffe, 0x000022a2,
187 0x2ae4, 0x00073ffe, 0x000022a2,
188 0x2ae4, 0x00073ffe, 0x000022a2,
189 0x240c, 0x000007ff, 0x00000000,
190 0x240c, 0x000007ff, 0x00000000,
191 0x240c, 0x000007ff, 0x00000000,
192 0x8a14, 0xf000001f, 0x00000007,
193 0x8a14, 0xf000001f, 0x00000007,
194 0x8a14, 0xf000001f, 0x00000007,
195 0x8b24, 0xffffffff, 0x00ffffff,
196 0x8b10, 0x0000ff0f, 0x00000000,
197 0x28a4c, 0x07ffffff, 0x4e000000,
198 0x28350, 0x3f3f3fff, 0x0000124a,
199 0x28350, 0x3f3f3fff, 0x0000124a,
200 0x28350, 0x3f3f3fff, 0x0000124a,
201 0x30, 0x000000ff, 0x0040,
202 0x34, 0x00000040, 0x00004040,
203 0x9100, 0x07ffffff, 0x03000000,
204 0x9100, 0x07ffffff, 0x03000000,
205 0x8e88, 0x01ff1f3f, 0x00000000,
206 0x8e88, 0x01ff1f3f, 0x00000000,
207 0x8e88, 0x01ff1f3f, 0x00000000,
208 0x8e84, 0x01ff1f3f, 0x00000000,
209 0x8e84, 0x01ff1f3f, 0x00000000,
210 0x8e84, 0x01ff1f3f, 0x00000000,
211 0x9060, 0x0000007f, 0x00000020,
212 0x9508, 0x00010000, 0x00010000,
213 0xac14, 0x000003ff, 0x00000003,
214 0xac14, 0x000003ff, 0x00000003,
215 0xac14, 0x000003ff, 0x00000003,
216 0xac10, 0xffffffff, 0x00000000,
217 0xac10, 0xffffffff, 0x00000000,
218 0xac10, 0xffffffff, 0x00000000,
219 0xac0c, 0xffffffff, 0x00001032,
220 0xac0c, 0xffffffff, 0x00001032,
221 0xac0c, 0xffffffff, 0x00001032,
222 0x88d4, 0x0000001f, 0x00000010,
223 0x88d4, 0x0000001f, 0x00000010,
224 0x88d4, 0x0000001f, 0x00000010,
225 0x15c0, 0x000c0fc0, 0x000c0400
226};
227
228static const u32 oland_golden_rlc_registers[] =
229{
230 0xc424, 0xffffffff, 0x00601005,
231 0xc47c, 0xffffffff, 0x10104040,
232 0xc488, 0xffffffff, 0x0100000a,
233 0xc314, 0xffffffff, 0x00000800,
234 0xc30c, 0xffffffff, 0x800000f4
235};
236
237static const u32 oland_golden_registers[] =
238{
239 0x9a10, 0x00010000, 0x00018208,
240 0x9830, 0xffffffff, 0x00000000,
241 0x9834, 0xf00fffff, 0x00000400,
242 0x9838, 0x0002021c, 0x00020200,
243 0xc78, 0x00000080, 0x00000000,
244 0xd030, 0x000300c0, 0x00800040,
245 0xd830, 0x000300c0, 0x00800040,
246 0x5bb0, 0x000000f0, 0x00000070,
247 0x5bc0, 0x00200000, 0x50100000,
248 0x7030, 0x31000311, 0x00000011,
249 0x2ae4, 0x00073ffe, 0x000022a2,
250 0x240c, 0x000007ff, 0x00000000,
251 0x8a14, 0xf000001f, 0x00000007,
252 0x8b24, 0xffffffff, 0x00ffffff,
253 0x8b10, 0x0000ff0f, 0x00000000,
254 0x28a4c, 0x07ffffff, 0x4e000000,
255 0x28350, 0x3f3f3fff, 0x00000082,
256 0x30, 0x000000ff, 0x0040,
257 0x34, 0x00000040, 0x00004040,
258 0x9100, 0x07ffffff, 0x03000000,
259 0x9060, 0x0000007f, 0x00000020,
260 0x9508, 0x00010000, 0x00010000,
261 0xac14, 0x000003ff, 0x000000f3,
262 0xac10, 0xffffffff, 0x00000000,
263 0xac0c, 0xffffffff, 0x00003210,
264 0x88d4, 0x0000001f, 0x00000010,
265 0x15c0, 0x000c0fc0, 0x000c0400
266};
267
268static const u32 tahiti_mgcg_cgcg_init[] =
269{
270 0xc400, 0xffffffff, 0xfffffffc,
271 0x802c, 0xffffffff, 0xe0000000,
272 0x9a60, 0xffffffff, 0x00000100,
273 0x92a4, 0xffffffff, 0x00000100,
274 0xc164, 0xffffffff, 0x00000100,
275 0x9774, 0xffffffff, 0x00000100,
276 0x8984, 0xffffffff, 0x06000100,
277 0x8a18, 0xffffffff, 0x00000100,
278 0x92a0, 0xffffffff, 0x00000100,
279 0xc380, 0xffffffff, 0x00000100,
280 0x8b28, 0xffffffff, 0x00000100,
281 0x9144, 0xffffffff, 0x00000100,
282 0x8d88, 0xffffffff, 0x00000100,
283 0x8d8c, 0xffffffff, 0x00000100,
284 0x9030, 0xffffffff, 0x00000100,
285 0x9034, 0xffffffff, 0x00000100,
286 0x9038, 0xffffffff, 0x00000100,
287 0x903c, 0xffffffff, 0x00000100,
288 0xad80, 0xffffffff, 0x00000100,
289 0xac54, 0xffffffff, 0x00000100,
290 0x897c, 0xffffffff, 0x06000100,
291 0x9868, 0xffffffff, 0x00000100,
292 0x9510, 0xffffffff, 0x00000100,
293 0xaf04, 0xffffffff, 0x00000100,
294 0xae04, 0xffffffff, 0x00000100,
295 0x949c, 0xffffffff, 0x00000100,
296 0x802c, 0xffffffff, 0xe0000000,
297 0x9160, 0xffffffff, 0x00010000,
298 0x9164, 0xffffffff, 0x00030002,
299 0x9168, 0xffffffff, 0x00040007,
300 0x916c, 0xffffffff, 0x00060005,
301 0x9170, 0xffffffff, 0x00090008,
302 0x9174, 0xffffffff, 0x00020001,
303 0x9178, 0xffffffff, 0x00040003,
304 0x917c, 0xffffffff, 0x00000007,
305 0x9180, 0xffffffff, 0x00060005,
306 0x9184, 0xffffffff, 0x00090008,
307 0x9188, 0xffffffff, 0x00030002,
308 0x918c, 0xffffffff, 0x00050004,
309 0x9190, 0xffffffff, 0x00000008,
310 0x9194, 0xffffffff, 0x00070006,
311 0x9198, 0xffffffff, 0x000a0009,
312 0x919c, 0xffffffff, 0x00040003,
313 0x91a0, 0xffffffff, 0x00060005,
314 0x91a4, 0xffffffff, 0x00000009,
315 0x91a8, 0xffffffff, 0x00080007,
316 0x91ac, 0xffffffff, 0x000b000a,
317 0x91b0, 0xffffffff, 0x00050004,
318 0x91b4, 0xffffffff, 0x00070006,
319 0x91b8, 0xffffffff, 0x0008000b,
320 0x91bc, 0xffffffff, 0x000a0009,
321 0x91c0, 0xffffffff, 0x000d000c,
322 0x91c4, 0xffffffff, 0x00060005,
323 0x91c8, 0xffffffff, 0x00080007,
324 0x91cc, 0xffffffff, 0x0000000b,
325 0x91d0, 0xffffffff, 0x000a0009,
326 0x91d4, 0xffffffff, 0x000d000c,
327 0x91d8, 0xffffffff, 0x00070006,
328 0x91dc, 0xffffffff, 0x00090008,
329 0x91e0, 0xffffffff, 0x0000000c,
330 0x91e4, 0xffffffff, 0x000b000a,
331 0x91e8, 0xffffffff, 0x000e000d,
332 0x91ec, 0xffffffff, 0x00080007,
333 0x91f0, 0xffffffff, 0x000a0009,
334 0x91f4, 0xffffffff, 0x0000000d,
335 0x91f8, 0xffffffff, 0x000c000b,
336 0x91fc, 0xffffffff, 0x000f000e,
337 0x9200, 0xffffffff, 0x00090008,
338 0x9204, 0xffffffff, 0x000b000a,
339 0x9208, 0xffffffff, 0x000c000f,
340 0x920c, 0xffffffff, 0x000e000d,
341 0x9210, 0xffffffff, 0x00110010,
342 0x9214, 0xffffffff, 0x000a0009,
343 0x9218, 0xffffffff, 0x000c000b,
344 0x921c, 0xffffffff, 0x0000000f,
345 0x9220, 0xffffffff, 0x000e000d,
346 0x9224, 0xffffffff, 0x00110010,
347 0x9228, 0xffffffff, 0x000b000a,
348 0x922c, 0xffffffff, 0x000d000c,
349 0x9230, 0xffffffff, 0x00000010,
350 0x9234, 0xffffffff, 0x000f000e,
351 0x9238, 0xffffffff, 0x00120011,
352 0x923c, 0xffffffff, 0x000c000b,
353 0x9240, 0xffffffff, 0x000e000d,
354 0x9244, 0xffffffff, 0x00000011,
355 0x9248, 0xffffffff, 0x0010000f,
356 0x924c, 0xffffffff, 0x00130012,
357 0x9250, 0xffffffff, 0x000d000c,
358 0x9254, 0xffffffff, 0x000f000e,
359 0x9258, 0xffffffff, 0x00100013,
360 0x925c, 0xffffffff, 0x00120011,
361 0x9260, 0xffffffff, 0x00150014,
362 0x9264, 0xffffffff, 0x000e000d,
363 0x9268, 0xffffffff, 0x0010000f,
364 0x926c, 0xffffffff, 0x00000013,
365 0x9270, 0xffffffff, 0x00120011,
366 0x9274, 0xffffffff, 0x00150014,
367 0x9278, 0xffffffff, 0x000f000e,
368 0x927c, 0xffffffff, 0x00110010,
369 0x9280, 0xffffffff, 0x00000014,
370 0x9284, 0xffffffff, 0x00130012,
371 0x9288, 0xffffffff, 0x00160015,
372 0x928c, 0xffffffff, 0x0010000f,
373 0x9290, 0xffffffff, 0x00120011,
374 0x9294, 0xffffffff, 0x00000015,
375 0x9298, 0xffffffff, 0x00140013,
376 0x929c, 0xffffffff, 0x00170016,
377 0x9150, 0xffffffff, 0x96940200,
378 0x8708, 0xffffffff, 0x00900100,
379 0xc478, 0xffffffff, 0x00000080,
380 0xc404, 0xffffffff, 0x0020003f,
381 0x30, 0xffffffff, 0x0000001c,
382 0x34, 0x000f0000, 0x000f0000,
383 0x160c, 0xffffffff, 0x00000100,
384 0x1024, 0xffffffff, 0x00000100,
385 0x102c, 0x00000101, 0x00000000,
386 0x20a8, 0xffffffff, 0x00000104,
387 0x264c, 0x000c0000, 0x000c0000,
388 0x2648, 0x000c0000, 0x000c0000,
389 0x55e4, 0xff000fff, 0x00000100,
390 0x55e8, 0x00000001, 0x00000001,
391 0x2f50, 0x00000001, 0x00000001,
392 0x30cc, 0xc0000fff, 0x00000104,
393 0xc1e4, 0x00000001, 0x00000001,
394 0xd0c0, 0xfffffff0, 0x00000100,
395 0xd8c0, 0xfffffff0, 0x00000100
396};
397
398static const u32 pitcairn_mgcg_cgcg_init[] =
399{
400 0xc400, 0xffffffff, 0xfffffffc,
401 0x802c, 0xffffffff, 0xe0000000,
402 0x9a60, 0xffffffff, 0x00000100,
403 0x92a4, 0xffffffff, 0x00000100,
404 0xc164, 0xffffffff, 0x00000100,
405 0x9774, 0xffffffff, 0x00000100,
406 0x8984, 0xffffffff, 0x06000100,
407 0x8a18, 0xffffffff, 0x00000100,
408 0x92a0, 0xffffffff, 0x00000100,
409 0xc380, 0xffffffff, 0x00000100,
410 0x8b28, 0xffffffff, 0x00000100,
411 0x9144, 0xffffffff, 0x00000100,
412 0x8d88, 0xffffffff, 0x00000100,
413 0x8d8c, 0xffffffff, 0x00000100,
414 0x9030, 0xffffffff, 0x00000100,
415 0x9034, 0xffffffff, 0x00000100,
416 0x9038, 0xffffffff, 0x00000100,
417 0x903c, 0xffffffff, 0x00000100,
418 0xad80, 0xffffffff, 0x00000100,
419 0xac54, 0xffffffff, 0x00000100,
420 0x897c, 0xffffffff, 0x06000100,
421 0x9868, 0xffffffff, 0x00000100,
422 0x9510, 0xffffffff, 0x00000100,
423 0xaf04, 0xffffffff, 0x00000100,
424 0xae04, 0xffffffff, 0x00000100,
425 0x949c, 0xffffffff, 0x00000100,
426 0x802c, 0xffffffff, 0xe0000000,
427 0x9160, 0xffffffff, 0x00010000,
428 0x9164, 0xffffffff, 0x00030002,
429 0x9168, 0xffffffff, 0x00040007,
430 0x916c, 0xffffffff, 0x00060005,
431 0x9170, 0xffffffff, 0x00090008,
432 0x9174, 0xffffffff, 0x00020001,
433 0x9178, 0xffffffff, 0x00040003,
434 0x917c, 0xffffffff, 0x00000007,
435 0x9180, 0xffffffff, 0x00060005,
436 0x9184, 0xffffffff, 0x00090008,
437 0x9188, 0xffffffff, 0x00030002,
438 0x918c, 0xffffffff, 0x00050004,
439 0x9190, 0xffffffff, 0x00000008,
440 0x9194, 0xffffffff, 0x00070006,
441 0x9198, 0xffffffff, 0x000a0009,
442 0x919c, 0xffffffff, 0x00040003,
443 0x91a0, 0xffffffff, 0x00060005,
444 0x91a4, 0xffffffff, 0x00000009,
445 0x91a8, 0xffffffff, 0x00080007,
446 0x91ac, 0xffffffff, 0x000b000a,
447 0x91b0, 0xffffffff, 0x00050004,
448 0x91b4, 0xffffffff, 0x00070006,
449 0x91b8, 0xffffffff, 0x0008000b,
450 0x91bc, 0xffffffff, 0x000a0009,
451 0x91c0, 0xffffffff, 0x000d000c,
452 0x9200, 0xffffffff, 0x00090008,
453 0x9204, 0xffffffff, 0x000b000a,
454 0x9208, 0xffffffff, 0x000c000f,
455 0x920c, 0xffffffff, 0x000e000d,
456 0x9210, 0xffffffff, 0x00110010,
457 0x9214, 0xffffffff, 0x000a0009,
458 0x9218, 0xffffffff, 0x000c000b,
459 0x921c, 0xffffffff, 0x0000000f,
460 0x9220, 0xffffffff, 0x000e000d,
461 0x9224, 0xffffffff, 0x00110010,
462 0x9228, 0xffffffff, 0x000b000a,
463 0x922c, 0xffffffff, 0x000d000c,
464 0x9230, 0xffffffff, 0x00000010,
465 0x9234, 0xffffffff, 0x000f000e,
466 0x9238, 0xffffffff, 0x00120011,
467 0x923c, 0xffffffff, 0x000c000b,
468 0x9240, 0xffffffff, 0x000e000d,
469 0x9244, 0xffffffff, 0x00000011,
470 0x9248, 0xffffffff, 0x0010000f,
471 0x924c, 0xffffffff, 0x00130012,
472 0x9250, 0xffffffff, 0x000d000c,
473 0x9254, 0xffffffff, 0x000f000e,
474 0x9258, 0xffffffff, 0x00100013,
475 0x925c, 0xffffffff, 0x00120011,
476 0x9260, 0xffffffff, 0x00150014,
477 0x9150, 0xffffffff, 0x96940200,
478 0x8708, 0xffffffff, 0x00900100,
479 0xc478, 0xffffffff, 0x00000080,
480 0xc404, 0xffffffff, 0x0020003f,
481 0x30, 0xffffffff, 0x0000001c,
482 0x34, 0x000f0000, 0x000f0000,
483 0x160c, 0xffffffff, 0x00000100,
484 0x1024, 0xffffffff, 0x00000100,
485 0x102c, 0x00000101, 0x00000000,
486 0x20a8, 0xffffffff, 0x00000104,
487 0x55e4, 0xff000fff, 0x00000100,
488 0x55e8, 0x00000001, 0x00000001,
489 0x2f50, 0x00000001, 0x00000001,
490 0x30cc, 0xc0000fff, 0x00000104,
491 0xc1e4, 0x00000001, 0x00000001,
492 0xd0c0, 0xfffffff0, 0x00000100,
493 0xd8c0, 0xfffffff0, 0x00000100
494};
495
496static const u32 verde_mgcg_cgcg_init[] =
497{
498 0xc400, 0xffffffff, 0xfffffffc,
499 0x802c, 0xffffffff, 0xe0000000,
500 0x9a60, 0xffffffff, 0x00000100,
501 0x92a4, 0xffffffff, 0x00000100,
502 0xc164, 0xffffffff, 0x00000100,
503 0x9774, 0xffffffff, 0x00000100,
504 0x8984, 0xffffffff, 0x06000100,
505 0x8a18, 0xffffffff, 0x00000100,
506 0x92a0, 0xffffffff, 0x00000100,
507 0xc380, 0xffffffff, 0x00000100,
508 0x8b28, 0xffffffff, 0x00000100,
509 0x9144, 0xffffffff, 0x00000100,
510 0x8d88, 0xffffffff, 0x00000100,
511 0x8d8c, 0xffffffff, 0x00000100,
512 0x9030, 0xffffffff, 0x00000100,
513 0x9034, 0xffffffff, 0x00000100,
514 0x9038, 0xffffffff, 0x00000100,
515 0x903c, 0xffffffff, 0x00000100,
516 0xad80, 0xffffffff, 0x00000100,
517 0xac54, 0xffffffff, 0x00000100,
518 0x897c, 0xffffffff, 0x06000100,
519 0x9868, 0xffffffff, 0x00000100,
520 0x9510, 0xffffffff, 0x00000100,
521 0xaf04, 0xffffffff, 0x00000100,
522 0xae04, 0xffffffff, 0x00000100,
523 0x949c, 0xffffffff, 0x00000100,
524 0x802c, 0xffffffff, 0xe0000000,
525 0x9160, 0xffffffff, 0x00010000,
526 0x9164, 0xffffffff, 0x00030002,
527 0x9168, 0xffffffff, 0x00040007,
528 0x916c, 0xffffffff, 0x00060005,
529 0x9170, 0xffffffff, 0x00090008,
530 0x9174, 0xffffffff, 0x00020001,
531 0x9178, 0xffffffff, 0x00040003,
532 0x917c, 0xffffffff, 0x00000007,
533 0x9180, 0xffffffff, 0x00060005,
534 0x9184, 0xffffffff, 0x00090008,
535 0x9188, 0xffffffff, 0x00030002,
536 0x918c, 0xffffffff, 0x00050004,
537 0x9190, 0xffffffff, 0x00000008,
538 0x9194, 0xffffffff, 0x00070006,
539 0x9198, 0xffffffff, 0x000a0009,
540 0x919c, 0xffffffff, 0x00040003,
541 0x91a0, 0xffffffff, 0x00060005,
542 0x91a4, 0xffffffff, 0x00000009,
543 0x91a8, 0xffffffff, 0x00080007,
544 0x91ac, 0xffffffff, 0x000b000a,
545 0x91b0, 0xffffffff, 0x00050004,
546 0x91b4, 0xffffffff, 0x00070006,
547 0x91b8, 0xffffffff, 0x0008000b,
548 0x91bc, 0xffffffff, 0x000a0009,
549 0x91c0, 0xffffffff, 0x000d000c,
550 0x9200, 0xffffffff, 0x00090008,
551 0x9204, 0xffffffff, 0x000b000a,
552 0x9208, 0xffffffff, 0x000c000f,
553 0x920c, 0xffffffff, 0x000e000d,
554 0x9210, 0xffffffff, 0x00110010,
555 0x9214, 0xffffffff, 0x000a0009,
556 0x9218, 0xffffffff, 0x000c000b,
557 0x921c, 0xffffffff, 0x0000000f,
558 0x9220, 0xffffffff, 0x000e000d,
559 0x9224, 0xffffffff, 0x00110010,
560 0x9228, 0xffffffff, 0x000b000a,
561 0x922c, 0xffffffff, 0x000d000c,
562 0x9230, 0xffffffff, 0x00000010,
563 0x9234, 0xffffffff, 0x000f000e,
564 0x9238, 0xffffffff, 0x00120011,
565 0x923c, 0xffffffff, 0x000c000b,
566 0x9240, 0xffffffff, 0x000e000d,
567 0x9244, 0xffffffff, 0x00000011,
568 0x9248, 0xffffffff, 0x0010000f,
569 0x924c, 0xffffffff, 0x00130012,
570 0x9250, 0xffffffff, 0x000d000c,
571 0x9254, 0xffffffff, 0x000f000e,
572 0x9258, 0xffffffff, 0x00100013,
573 0x925c, 0xffffffff, 0x00120011,
574 0x9260, 0xffffffff, 0x00150014,
575 0x9150, 0xffffffff, 0x96940200,
576 0x8708, 0xffffffff, 0x00900100,
577 0xc478, 0xffffffff, 0x00000080,
578 0xc404, 0xffffffff, 0x0020003f,
579 0x30, 0xffffffff, 0x0000001c,
580 0x34, 0x000f0000, 0x000f0000,
581 0x160c, 0xffffffff, 0x00000100,
582 0x1024, 0xffffffff, 0x00000100,
583 0x102c, 0x00000101, 0x00000000,
584 0x20a8, 0xffffffff, 0x00000104,
585 0x264c, 0x000c0000, 0x000c0000,
586 0x2648, 0x000c0000, 0x000c0000,
587 0x55e4, 0xff000fff, 0x00000100,
588 0x55e8, 0x00000001, 0x00000001,
589 0x2f50, 0x00000001, 0x00000001,
590 0x30cc, 0xc0000fff, 0x00000104,
591 0xc1e4, 0x00000001, 0x00000001,
592 0xd0c0, 0xfffffff0, 0x00000100,
593 0xd8c0, 0xfffffff0, 0x00000100
594};
595
596static const u32 oland_mgcg_cgcg_init[] =
597{
598 0xc400, 0xffffffff, 0xfffffffc,
599 0x802c, 0xffffffff, 0xe0000000,
600 0x9a60, 0xffffffff, 0x00000100,
601 0x92a4, 0xffffffff, 0x00000100,
602 0xc164, 0xffffffff, 0x00000100,
603 0x9774, 0xffffffff, 0x00000100,
604 0x8984, 0xffffffff, 0x06000100,
605 0x8a18, 0xffffffff, 0x00000100,
606 0x92a0, 0xffffffff, 0x00000100,
607 0xc380, 0xffffffff, 0x00000100,
608 0x8b28, 0xffffffff, 0x00000100,
609 0x9144, 0xffffffff, 0x00000100,
610 0x8d88, 0xffffffff, 0x00000100,
611 0x8d8c, 0xffffffff, 0x00000100,
612 0x9030, 0xffffffff, 0x00000100,
613 0x9034, 0xffffffff, 0x00000100,
614 0x9038, 0xffffffff, 0x00000100,
615 0x903c, 0xffffffff, 0x00000100,
616 0xad80, 0xffffffff, 0x00000100,
617 0xac54, 0xffffffff, 0x00000100,
618 0x897c, 0xffffffff, 0x06000100,
619 0x9868, 0xffffffff, 0x00000100,
620 0x9510, 0xffffffff, 0x00000100,
621 0xaf04, 0xffffffff, 0x00000100,
622 0xae04, 0xffffffff, 0x00000100,
623 0x949c, 0xffffffff, 0x00000100,
624 0x802c, 0xffffffff, 0xe0000000,
625 0x9160, 0xffffffff, 0x00010000,
626 0x9164, 0xffffffff, 0x00030002,
627 0x9168, 0xffffffff, 0x00040007,
628 0x916c, 0xffffffff, 0x00060005,
629 0x9170, 0xffffffff, 0x00090008,
630 0x9174, 0xffffffff, 0x00020001,
631 0x9178, 0xffffffff, 0x00040003,
632 0x917c, 0xffffffff, 0x00000007,
633 0x9180, 0xffffffff, 0x00060005,
634 0x9184, 0xffffffff, 0x00090008,
635 0x9188, 0xffffffff, 0x00030002,
636 0x918c, 0xffffffff, 0x00050004,
637 0x9190, 0xffffffff, 0x00000008,
638 0x9194, 0xffffffff, 0x00070006,
639 0x9198, 0xffffffff, 0x000a0009,
640 0x919c, 0xffffffff, 0x00040003,
641 0x91a0, 0xffffffff, 0x00060005,
642 0x91a4, 0xffffffff, 0x00000009,
643 0x91a8, 0xffffffff, 0x00080007,
644 0x91ac, 0xffffffff, 0x000b000a,
645 0x91b0, 0xffffffff, 0x00050004,
646 0x91b4, 0xffffffff, 0x00070006,
647 0x91b8, 0xffffffff, 0x0008000b,
648 0x91bc, 0xffffffff, 0x000a0009,
649 0x91c0, 0xffffffff, 0x000d000c,
650 0x91c4, 0xffffffff, 0x00060005,
651 0x91c8, 0xffffffff, 0x00080007,
652 0x91cc, 0xffffffff, 0x0000000b,
653 0x91d0, 0xffffffff, 0x000a0009,
654 0x91d4, 0xffffffff, 0x000d000c,
655 0x9150, 0xffffffff, 0x96940200,
656 0x8708, 0xffffffff, 0x00900100,
657 0xc478, 0xffffffff, 0x00000080,
658 0xc404, 0xffffffff, 0x0020003f,
659 0x30, 0xffffffff, 0x0000001c,
660 0x34, 0x000f0000, 0x000f0000,
661 0x160c, 0xffffffff, 0x00000100,
662 0x1024, 0xffffffff, 0x00000100,
663 0x102c, 0x00000101, 0x00000000,
664 0x20a8, 0xffffffff, 0x00000104,
665 0x264c, 0x000c0000, 0x000c0000,
666 0x2648, 0x000c0000, 0x000c0000,
667 0x55e4, 0xff000fff, 0x00000100,
668 0x55e8, 0x00000001, 0x00000001,
669 0x2f50, 0x00000001, 0x00000001,
670 0x30cc, 0xc0000fff, 0x00000104,
671 0xc1e4, 0x00000001, 0x00000001,
672 0xd0c0, 0xfffffff0, 0x00000100,
673 0xd8c0, 0xfffffff0, 0x00000100
674};
675
676static u32 verde_pg_init[] =
677{
678 0x353c, 0xffffffff, 0x40000,
679 0x3538, 0xffffffff, 0x200010ff,
680 0x353c, 0xffffffff, 0x0,
681 0x353c, 0xffffffff, 0x0,
682 0x353c, 0xffffffff, 0x0,
683 0x353c, 0xffffffff, 0x0,
684 0x353c, 0xffffffff, 0x0,
685 0x353c, 0xffffffff, 0x7007,
686 0x3538, 0xffffffff, 0x300010ff,
687 0x353c, 0xffffffff, 0x0,
688 0x353c, 0xffffffff, 0x0,
689 0x353c, 0xffffffff, 0x0,
690 0x353c, 0xffffffff, 0x0,
691 0x353c, 0xffffffff, 0x0,
692 0x353c, 0xffffffff, 0x400000,
693 0x3538, 0xffffffff, 0x100010ff,
694 0x353c, 0xffffffff, 0x0,
695 0x353c, 0xffffffff, 0x0,
696 0x353c, 0xffffffff, 0x0,
697 0x353c, 0xffffffff, 0x0,
698 0x353c, 0xffffffff, 0x0,
699 0x353c, 0xffffffff, 0x120200,
700 0x3538, 0xffffffff, 0x500010ff,
701 0x353c, 0xffffffff, 0x0,
702 0x353c, 0xffffffff, 0x0,
703 0x353c, 0xffffffff, 0x0,
704 0x353c, 0xffffffff, 0x0,
705 0x353c, 0xffffffff, 0x0,
706 0x353c, 0xffffffff, 0x1e1e16,
707 0x3538, 0xffffffff, 0x600010ff,
708 0x353c, 0xffffffff, 0x0,
709 0x353c, 0xffffffff, 0x0,
710 0x353c, 0xffffffff, 0x0,
711 0x353c, 0xffffffff, 0x0,
712 0x353c, 0xffffffff, 0x0,
713 0x353c, 0xffffffff, 0x171f1e,
714 0x3538, 0xffffffff, 0x700010ff,
715 0x353c, 0xffffffff, 0x0,
716 0x353c, 0xffffffff, 0x0,
717 0x353c, 0xffffffff, 0x0,
718 0x353c, 0xffffffff, 0x0,
719 0x353c, 0xffffffff, 0x0,
720 0x353c, 0xffffffff, 0x0,
721 0x3538, 0xffffffff, 0x9ff,
722 0x3500, 0xffffffff, 0x0,
723 0x3504, 0xffffffff, 0x10000800,
724 0x3504, 0xffffffff, 0xf,
725 0x3504, 0xffffffff, 0xf,
726 0x3500, 0xffffffff, 0x4,
727 0x3504, 0xffffffff, 0x1000051e,
728 0x3504, 0xffffffff, 0xffff,
729 0x3504, 0xffffffff, 0xffff,
730 0x3500, 0xffffffff, 0x8,
731 0x3504, 0xffffffff, 0x80500,
732 0x3500, 0xffffffff, 0x12,
733 0x3504, 0xffffffff, 0x9050c,
734 0x3500, 0xffffffff, 0x1d,
735 0x3504, 0xffffffff, 0xb052c,
736 0x3500, 0xffffffff, 0x2a,
737 0x3504, 0xffffffff, 0x1053e,
738 0x3500, 0xffffffff, 0x2d,
739 0x3504, 0xffffffff, 0x10546,
740 0x3500, 0xffffffff, 0x30,
741 0x3504, 0xffffffff, 0xa054e,
742 0x3500, 0xffffffff, 0x3c,
743 0x3504, 0xffffffff, 0x1055f,
744 0x3500, 0xffffffff, 0x3f,
745 0x3504, 0xffffffff, 0x10567,
746 0x3500, 0xffffffff, 0x42,
747 0x3504, 0xffffffff, 0x1056f,
748 0x3500, 0xffffffff, 0x45,
749 0x3504, 0xffffffff, 0x10572,
750 0x3500, 0xffffffff, 0x48,
751 0x3504, 0xffffffff, 0x20575,
752 0x3500, 0xffffffff, 0x4c,
753 0x3504, 0xffffffff, 0x190801,
754 0x3500, 0xffffffff, 0x67,
755 0x3504, 0xffffffff, 0x1082a,
756 0x3500, 0xffffffff, 0x6a,
757 0x3504, 0xffffffff, 0x1b082d,
758 0x3500, 0xffffffff, 0x87,
759 0x3504, 0xffffffff, 0x310851,
760 0x3500, 0xffffffff, 0xba,
761 0x3504, 0xffffffff, 0x891,
762 0x3500, 0xffffffff, 0xbc,
763 0x3504, 0xffffffff, 0x893,
764 0x3500, 0xffffffff, 0xbe,
765 0x3504, 0xffffffff, 0x20895,
766 0x3500, 0xffffffff, 0xc2,
767 0x3504, 0xffffffff, 0x20899,
768 0x3500, 0xffffffff, 0xc6,
769 0x3504, 0xffffffff, 0x2089d,
770 0x3500, 0xffffffff, 0xca,
771 0x3504, 0xffffffff, 0x8a1,
772 0x3500, 0xffffffff, 0xcc,
773 0x3504, 0xffffffff, 0x8a3,
774 0x3500, 0xffffffff, 0xce,
775 0x3504, 0xffffffff, 0x308a5,
776 0x3500, 0xffffffff, 0xd3,
777 0x3504, 0xffffffff, 0x6d08cd,
778 0x3500, 0xffffffff, 0x142,
779 0x3504, 0xffffffff, 0x2000095a,
780 0x3504, 0xffffffff, 0x1,
781 0x3500, 0xffffffff, 0x144,
782 0x3504, 0xffffffff, 0x301f095b,
783 0x3500, 0xffffffff, 0x165,
784 0x3504, 0xffffffff, 0xc094d,
785 0x3500, 0xffffffff, 0x173,
786 0x3504, 0xffffffff, 0xf096d,
787 0x3500, 0xffffffff, 0x184,
788 0x3504, 0xffffffff, 0x15097f,
789 0x3500, 0xffffffff, 0x19b,
790 0x3504, 0xffffffff, 0xc0998,
791 0x3500, 0xffffffff, 0x1a9,
792 0x3504, 0xffffffff, 0x409a7,
793 0x3500, 0xffffffff, 0x1af,
794 0x3504, 0xffffffff, 0xcdc,
795 0x3500, 0xffffffff, 0x1b1,
796 0x3504, 0xffffffff, 0x800,
797 0x3508, 0xffffffff, 0x6c9b2000,
798 0x3510, 0xfc00, 0x2000,
799 0x3544, 0xffffffff, 0xfc0,
800 0x28d4, 0x00000100, 0x100
801};
802
803static void si_init_golden_registers(struct radeon_device *rdev)
804{
805 switch (rdev->family) {
806 case CHIP_TAHITI:
807 radeon_program_register_sequence(rdev,
808 tahiti_golden_registers,
809 (const u32)ARRAY_SIZE(tahiti_golden_registers));
810 radeon_program_register_sequence(rdev,
811 tahiti_golden_rlc_registers,
812 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
813 radeon_program_register_sequence(rdev,
814 tahiti_mgcg_cgcg_init,
815 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
816 radeon_program_register_sequence(rdev,
817 tahiti_golden_registers2,
818 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
819 break;
820 case CHIP_PITCAIRN:
821 radeon_program_register_sequence(rdev,
822 pitcairn_golden_registers,
823 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
824 radeon_program_register_sequence(rdev,
825 pitcairn_golden_rlc_registers,
826 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
827 radeon_program_register_sequence(rdev,
828 pitcairn_mgcg_cgcg_init,
829 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
830 break;
831 case CHIP_VERDE:
832 radeon_program_register_sequence(rdev,
833 verde_golden_registers,
834 (const u32)ARRAY_SIZE(verde_golden_registers));
835 radeon_program_register_sequence(rdev,
836 verde_golden_rlc_registers,
837 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
838 radeon_program_register_sequence(rdev,
839 verde_mgcg_cgcg_init,
840 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
841 radeon_program_register_sequence(rdev,
842 verde_pg_init,
843 (const u32)ARRAY_SIZE(verde_pg_init));
844 break;
845 case CHIP_OLAND:
846 radeon_program_register_sequence(rdev,
847 oland_golden_registers,
848 (const u32)ARRAY_SIZE(oland_golden_registers));
849 radeon_program_register_sequence(rdev,
850 oland_golden_rlc_registers,
851 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
852 radeon_program_register_sequence(rdev,
853 oland_mgcg_cgcg_init,
854 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
855 break;
856 default:
857 break;
858 }
859}
860
454d2e2a
AD		 861#define PCIE_BUS_CLK 10000
862#define TCLK (PCIE_BUS_CLK / 10)
863
864/**
865 * si_get_xclk - get the xclk
866 *
867 * @rdev: radeon_device pointer
868 *
869 * Returns the reference clock used by the gfx engine
870 * (SI).
871 */
872u32 si_get_xclk(struct radeon_device *rdev)
873{
874 u32 reference_clock = rdev->clock.spll.reference_freq;
875 u32 tmp;
876
877 tmp = RREG32(CG_CLKPIN_CNTL_2);
878 if (tmp & MUX_TCLK_TO_XCLK)
879 return TCLK;
880
881 tmp = RREG32(CG_CLKPIN_CNTL);
882 if (tmp & XTALIN_DIVIDE)
883 return reference_clock / 4;
884
885 return reference_clock;
886}
887
1bd47d2e
AD		 888/* get temperature in millidegrees */
889int si_get_temp(struct radeon_device *rdev)
890{
891 u32 temp;
892 int actual_temp = 0;
893
894 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
895 CTF_TEMP_SHIFT;
896
897 if (temp & 0x200)
898 actual_temp = 255;
899 else
900 actual_temp = temp & 0x1ff;
901
902 actual_temp = (actual_temp * 1000);
903
904 return actual_temp;
905}
906
8b074dd6
AD		 907#define TAHITI_IO_MC_REGS_SIZE 36
908
909static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
910 {0x0000006f, 0x03044000},
911 {0x00000070, 0x0480c018},
912 {0x00000071, 0x00000040},
913 {0x00000072, 0x01000000},
914 {0x00000074, 0x000000ff},
915 {0x00000075, 0x00143400},
916 {0x00000076, 0x08ec0800},
917 {0x00000077, 0x040000cc},
918 {0x00000079, 0x00000000},
919 {0x0000007a, 0x21000409},
920 {0x0000007c, 0x00000000},
921 {0x0000007d, 0xe8000000},
922 {0x0000007e, 0x044408a8},
923 {0x0000007f, 0x00000003},
924 {0x00000080, 0x00000000},
925 {0x00000081, 0x01000000},
926 {0x00000082, 0x02000000},
927 {0x00000083, 0x00000000},
928 {0x00000084, 0xe3f3e4f4},
929 {0x00000085, 0x00052024},
930 {0x00000087, 0x00000000},
931 {0x00000088, 0x66036603},
932 {0x00000089, 0x01000000},
933 {0x0000008b, 0x1c0a0000},
934 {0x0000008c, 0xff010000},
935 {0x0000008e, 0xffffefff},
936 {0x0000008f, 0xfff3efff},
937 {0x00000090, 0xfff3efbf},
938 {0x00000094, 0x00101101},
939 {0x00000095, 0x00000fff},
940 {0x00000096, 0x00116fff},
941 {0x00000097, 0x60010000},
942 {0x00000098, 0x10010000},
943 {0x00000099, 0x00006000},
944 {0x0000009a, 0x00001000},
945 {0x0000009f, 0x00a77400}
946};
947
948static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
949 {0x0000006f, 0x03044000},
950 {0x00000070, 0x0480c018},
951 {0x00000071, 0x00000040},
952 {0x00000072, 0x01000000},
953 {0x00000074, 0x000000ff},
954 {0x00000075, 0x00143400},
955 {0x00000076, 0x08ec0800},
956 {0x00000077, 0x040000cc},
957 {0x00000079, 0x00000000},
958 {0x0000007a, 0x21000409},
959 {0x0000007c, 0x00000000},
960 {0x0000007d, 0xe8000000},
961 {0x0000007e, 0x044408a8},
962 {0x0000007f, 0x00000003},
963 {0x00000080, 0x00000000},
964 {0x00000081, 0x01000000},
965 {0x00000082, 0x02000000},
966 {0x00000083, 0x00000000},
967 {0x00000084, 0xe3f3e4f4},
968 {0x00000085, 0x00052024},
969 {0x00000087, 0x00000000},
970 {0x00000088, 0x66036603},
971 {0x00000089, 0x01000000},
972 {0x0000008b, 0x1c0a0000},
973 {0x0000008c, 0xff010000},
974 {0x0000008e, 0xffffefff},
975 {0x0000008f, 0xfff3efff},
976 {0x00000090, 0xfff3efbf},
977 {0x00000094, 0x00101101},
978 {0x00000095, 0x00000fff},
979 {0x00000096, 0x00116fff},
980 {0x00000097, 0x60010000},
981 {0x00000098, 0x10010000},
982 {0x00000099, 0x00006000},
983 {0x0000009a, 0x00001000},
984 {0x0000009f, 0x00a47400}
985};
986
987static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
988 {0x0000006f, 0x03044000},
989 {0x00000070, 0x0480c018},
990 {0x00000071, 0x00000040},
991 {0x00000072, 0x01000000},
992 {0x00000074, 0x000000ff},
993 {0x00000075, 0x00143400},
994 {0x00000076, 0x08ec0800},
995 {0x00000077, 0x040000cc},
996 {0x00000079, 0x00000000},
997 {0x0000007a, 0x21000409},
998 {0x0000007c, 0x00000000},
999 {0x0000007d, 0xe8000000},
1000 {0x0000007e, 0x044408a8},
1001 {0x0000007f, 0x00000003},
1002 {0x00000080, 0x00000000},
1003 {0x00000081, 0x01000000},
1004 {0x00000082, 0x02000000},
1005 {0x00000083, 0x00000000},
1006 {0x00000084, 0xe3f3e4f4},
1007 {0x00000085, 0x00052024},
1008 {0x00000087, 0x00000000},
1009 {0x00000088, 0x66036603},
1010 {0x00000089, 0x01000000},
1011 {0x0000008b, 0x1c0a0000},
1012 {0x0000008c, 0xff010000},
1013 {0x0000008e, 0xffffefff},
1014 {0x0000008f, 0xfff3efff},
1015 {0x00000090, 0xfff3efbf},
1016 {0x00000094, 0x00101101},
1017 {0x00000095, 0x00000fff},
1018 {0x00000096, 0x00116fff},
1019 {0x00000097, 0x60010000},
1020 {0x00000098, 0x10010000},
1021 {0x00000099, 0x00006000},
1022 {0x0000009a, 0x00001000},
1023 {0x0000009f, 0x00a37400}
1024};
1025
bcc7f5d2
AD		 1026static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1027 {0x0000006f, 0x03044000},
1028 {0x00000070, 0x0480c018},
1029 {0x00000071, 0x00000040},
1030 {0x00000072, 0x01000000},
1031 {0x00000074, 0x000000ff},
1032 {0x00000075, 0x00143400},
1033 {0x00000076, 0x08ec0800},
1034 {0x00000077, 0x040000cc},
1035 {0x00000079, 0x00000000},
1036 {0x0000007a, 0x21000409},
1037 {0x0000007c, 0x00000000},
1038 {0x0000007d, 0xe8000000},
1039 {0x0000007e, 0x044408a8},
1040 {0x0000007f, 0x00000003},
1041 {0x00000080, 0x00000000},
1042 {0x00000081, 0x01000000},
1043 {0x00000082, 0x02000000},
1044 {0x00000083, 0x00000000},
1045 {0x00000084, 0xe3f3e4f4},
1046 {0x00000085, 0x00052024},
1047 {0x00000087, 0x00000000},
1048 {0x00000088, 0x66036603},
1049 {0x00000089, 0x01000000},
1050 {0x0000008b, 0x1c0a0000},
1051 {0x0000008c, 0xff010000},
1052 {0x0000008e, 0xffffefff},
1053 {0x0000008f, 0xfff3efff},
1054 {0x00000090, 0xfff3efbf},
1055 {0x00000094, 0x00101101},
1056 {0x00000095, 0x00000fff},
1057 {0x00000096, 0x00116fff},
1058 {0x00000097, 0x60010000},
1059 {0x00000098, 0x10010000},
1060 {0x00000099, 0x00006000},
1061 {0x0000009a, 0x00001000},
1062 {0x0000009f, 0x00a17730}
1063};
1064
8b074dd6
AD		 1065/* ucode loading */
1066static int si_mc_load_microcode(struct radeon_device *rdev)
1067{
1068 const __be32 *fw_data;
1069 u32 running, blackout = 0;
1070 u32 *io_mc_regs;
1071 int i, ucode_size, regs_size;
1072
1073 if (!rdev->mc_fw)
1074 return -EINVAL;
1075
1076 switch (rdev->family) {
1077 case CHIP_TAHITI:
1078 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1079 ucode_size = SI_MC_UCODE_SIZE;
1080 regs_size = TAHITI_IO_MC_REGS_SIZE;
1081 break;
1082 case CHIP_PITCAIRN:
1083 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1084 ucode_size = SI_MC_UCODE_SIZE;
1085 regs_size = TAHITI_IO_MC_REGS_SIZE;
1086 break;
1087 case CHIP_VERDE:
1088 default:
1089 io_mc_regs = (u32 *)&verde_io_mc_regs;
1090 ucode_size = SI_MC_UCODE_SIZE;
1091 regs_size = TAHITI_IO_MC_REGS_SIZE;
1092 break;
bcc7f5d2
AD		 1093 case CHIP_OLAND:
1094 io_mc_regs = (u32 *)&oland_io_mc_regs;
1095 ucode_size = OLAND_MC_UCODE_SIZE;
1096 regs_size = TAHITI_IO_MC_REGS_SIZE;
1097 break;
8b074dd6
AD		 1098 }
1099
1100 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1101
1102 if (running == 0) {
1103 if (running) {
1104 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1105 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1106 }
1107
1108 /* reset the engine and set to writable */
1109 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1110 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1111
1112 /* load mc io regs */
1113 for (i = 0; i < regs_size; i++) {
1114 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1115 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1116 }
1117 /* load the MC ucode */
1118 fw_data = (const __be32 *)rdev->mc_fw->data;
1119 for (i = 0; i < ucode_size; i++)
1120 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1121
1122 /* put the engine back into the active state */
1123 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1124 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1125 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1126
1127 /* wait for training to complete */
1128 for (i = 0; i < rdev->usec_timeout; i++) {
1129 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1130 break;
1131 udelay(1);
1132 }
1133 for (i = 0; i < rdev->usec_timeout; i++) {
1134 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1135 break;
1136 udelay(1);
1137 }
1138
1139 if (running)
1140 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1141 }
1142
1143 return 0;
1144}
1145
0f0de06c
AD		 1146static int si_init_microcode(struct radeon_device *rdev)
1147{
1148 struct platform_device *pdev;
1149 const char *chip_name;
1150 const char *rlc_chip_name;
1151 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1152 char fw_name[30];
1153 int err;
1154
1155 DRM_DEBUG("\n");
1156
1157 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1158 err = IS_ERR(pdev);
1159 if (err) {
1160 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1161 return -EINVAL;
1162 }
1163
1164 switch (rdev->family) {
1165 case CHIP_TAHITI:
1166 chip_name = "TAHITI";
1167 rlc_chip_name = "TAHITI";
1168 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1169 me_req_size = SI_PM4_UCODE_SIZE * 4;
1170 ce_req_size = SI_CE_UCODE_SIZE * 4;
1171 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1172 mc_req_size = SI_MC_UCODE_SIZE * 4;
1173 break;
1174 case CHIP_PITCAIRN:
1175 chip_name = "PITCAIRN";
1176 rlc_chip_name = "PITCAIRN";
1177 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1178 me_req_size = SI_PM4_UCODE_SIZE * 4;
1179 ce_req_size = SI_CE_UCODE_SIZE * 4;
1180 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1181 mc_req_size = SI_MC_UCODE_SIZE * 4;
1182 break;
1183 case CHIP_VERDE:
1184 chip_name = "VERDE";
1185 rlc_chip_name = "VERDE";
1186 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1187 me_req_size = SI_PM4_UCODE_SIZE * 4;
1188 ce_req_size = SI_CE_UCODE_SIZE * 4;
1189 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1190 mc_req_size = SI_MC_UCODE_SIZE * 4;
1191 break;
bcc7f5d2
AD		 1192 case CHIP_OLAND:
1193 chip_name = "OLAND";
1194 rlc_chip_name = "OLAND";
1195 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1196 me_req_size = SI_PM4_UCODE_SIZE * 4;
1197 ce_req_size = SI_CE_UCODE_SIZE * 4;
1198 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1199 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1200 break;
0f0de06c
AD		 1201 default: BUG();
1202 }
1203
1204 DRM_INFO("Loading %s Microcode\n", chip_name);
1205
1206 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1207 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1208 if (err)
1209 goto out;
1210 if (rdev->pfp_fw->size != pfp_req_size) {
1211 printk(KERN_ERR
1212 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1213 rdev->pfp_fw->size, fw_name);
1214 err = -EINVAL;
1215 goto out;
1216 }
1217
1218 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1219 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1220 if (err)
1221 goto out;
1222 if (rdev->me_fw->size != me_req_size) {
1223 printk(KERN_ERR
1224 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1225 rdev->me_fw->size, fw_name);
1226 err = -EINVAL;
1227 }
1228
1229 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1230 err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
1231 if (err)
1232 goto out;
1233 if (rdev->ce_fw->size != ce_req_size) {
1234 printk(KERN_ERR
1235 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1236 rdev->ce_fw->size, fw_name);
1237 err = -EINVAL;
1238 }
1239
1240 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1241 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1242 if (err)
1243 goto out;
1244 if (rdev->rlc_fw->size != rlc_req_size) {
1245 printk(KERN_ERR
1246 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1247 rdev->rlc_fw->size, fw_name);
1248 err = -EINVAL;
1249 }
1250
1251 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1252 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
1253 if (err)
1254 goto out;
1255 if (rdev->mc_fw->size != mc_req_size) {
1256 printk(KERN_ERR
1257 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1258 rdev->mc_fw->size, fw_name);
1259 err = -EINVAL;
1260 }
1261
1262out:
1263 platform_device_unregister(pdev);
1264
1265 if (err) {
1266 if (err != -EINVAL)
1267 printk(KERN_ERR
1268 "si_cp: Failed to load firmware \"%s\"\n",
1269 fw_name);
1270 release_firmware(rdev->pfp_fw);
1271 rdev->pfp_fw = NULL;
1272 release_firmware(rdev->me_fw);
1273 rdev->me_fw = NULL;
1274 release_firmware(rdev->ce_fw);
1275 rdev->ce_fw = NULL;
1276 release_firmware(rdev->rlc_fw);
1277 rdev->rlc_fw = NULL;
1278 release_firmware(rdev->mc_fw);
1279 rdev->mc_fw = NULL;
1280 }
1281 return err;
1282}
1283
43b3cd99
AD		 1284/* watermark setup */
1285static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1286 struct radeon_crtc *radeon_crtc,
1287 struct drm_display_mode *mode,
1288 struct drm_display_mode *other_mode)
1289{
1290 u32 tmp;
1291 /*
1292 * Line Buffer Setup
1293 * There are 3 line buffers, each one shared by 2 display controllers.
1294 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1295 * the display controllers. The paritioning is done via one of four
1296 * preset allocations specified in bits 21:20:
1297 * 0 - half lb
1298 * 2 - whole lb, other crtc must be disabled
1299 */
1300 /* this can get tricky if we have two large displays on a paired group
1301 * of crtcs. Ideally for multiple large displays we'd assign them to
1302 * non-linked crtcs for maximum line buffer allocation.
1303 */
1304 if (radeon_crtc->base.enabled && mode) {
1305 if (other_mode)
1306 tmp = 0; /* 1/2 */
1307 else
1308 tmp = 2; /* whole */
1309 } else
1310 tmp = 0;
1311
1312 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1313 DC_LB_MEMORY_CONFIG(tmp));
1314
1315 if (radeon_crtc->base.enabled && mode) {
1316 switch (tmp) {
1317 case 0:
1318 default:
1319 return 4096 * 2;
1320 case 2:
1321 return 8192 * 2;
1322 }
1323 }
1324
1325 /* controller not enabled, so no lb used */
1326 return 0;
1327}
1328
ca7db22b 1329static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1330{
1331 u32 tmp = RREG32(MC_SHARED_CHMAP);
1332
1333 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1334 case 0:
1335 default:
1336 return 1;
1337 case 1:
1338 return 2;
1339 case 2:
1340 return 4;
1341 case 3:
1342 return 8;
1343 case 4:
1344 return 3;
1345 case 5:
1346 return 6;
1347 case 6:
1348 return 10;
1349 case 7:
1350 return 12;
1351 case 8:
1352 return 16;
1353 }
1354}
1355
1356struct dce6_wm_params {
1357 u32 dram_channels; /* number of dram channels */
1358 u32 yclk; /* bandwidth per dram data pin in kHz */
1359 u32 sclk; /* engine clock in kHz */
1360 u32 disp_clk; /* display clock in kHz */
1361 u32 src_width; /* viewport width */
1362 u32 active_time; /* active display time in ns */
1363 u32 blank_time; /* blank time in ns */
1364 bool interlaced; /* mode is interlaced */
1365 fixed20_12 vsc; /* vertical scale ratio */
1366 u32 num_heads; /* number of active crtcs */
1367 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1368 u32 lb_size; /* line buffer allocated to pipe */
1369 u32 vtaps; /* vertical scaler taps */
1370};
1371
1372static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1373{
1374 /* Calculate raw DRAM Bandwidth */
1375 fixed20_12 dram_efficiency; /* 0.7 */
1376 fixed20_12 yclk, dram_channels, bandwidth;
1377 fixed20_12 a;
1378
1379 a.full = dfixed_const(1000);
1380 yclk.full = dfixed_const(wm->yclk);
1381 yclk.full = dfixed_div(yclk, a);
1382 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1383 a.full = dfixed_const(10);
1384 dram_efficiency.full = dfixed_const(7);
1385 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1386 bandwidth.full = dfixed_mul(dram_channels, yclk);
1387 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1388
1389 return dfixed_trunc(bandwidth);
1390}
1391
1392static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1393{
1394 /* Calculate DRAM Bandwidth and the part allocated to display. */
1395 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1396 fixed20_12 yclk, dram_channels, bandwidth;
1397 fixed20_12 a;
1398
1399 a.full = dfixed_const(1000);
1400 yclk.full = dfixed_const(wm->yclk);
1401 yclk.full = dfixed_div(yclk, a);
1402 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1403 a.full = dfixed_const(10);
1404 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1405 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1406 bandwidth.full = dfixed_mul(dram_channels, yclk);
1407 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1408
1409 return dfixed_trunc(bandwidth);
1410}
1411
1412static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1413{
1414 /* Calculate the display Data return Bandwidth */
1415 fixed20_12 return_efficiency; /* 0.8 */
1416 fixed20_12 sclk, bandwidth;
1417 fixed20_12 a;
1418
1419 a.full = dfixed_const(1000);
1420 sclk.full = dfixed_const(wm->sclk);
1421 sclk.full = dfixed_div(sclk, a);
1422 a.full = dfixed_const(10);
1423 return_efficiency.full = dfixed_const(8);
1424 return_efficiency.full = dfixed_div(return_efficiency, a);
1425 a.full = dfixed_const(32);
1426 bandwidth.full = dfixed_mul(a, sclk);
1427 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1428
1429 return dfixed_trunc(bandwidth);
1430}
1431
1432static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1433{
1434 return 32;
1435}
1436
1437static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1438{
1439 /* Calculate the DMIF Request Bandwidth */
1440 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1441 fixed20_12 disp_clk, sclk, bandwidth;
1442 fixed20_12 a, b1, b2;
1443 u32 min_bandwidth;
1444
1445 a.full = dfixed_const(1000);
1446 disp_clk.full = dfixed_const(wm->disp_clk);
1447 disp_clk.full = dfixed_div(disp_clk, a);
1448 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1449 b1.full = dfixed_mul(a, disp_clk);
1450
1451 a.full = dfixed_const(1000);
1452 sclk.full = dfixed_const(wm->sclk);
1453 sclk.full = dfixed_div(sclk, a);
1454 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1455 b2.full = dfixed_mul(a, sclk);
1456
1457 a.full = dfixed_const(10);
1458 disp_clk_request_efficiency.full = dfixed_const(8);
1459 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1460
1461 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1462
1463 a.full = dfixed_const(min_bandwidth);
1464 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1465
1466 return dfixed_trunc(bandwidth);
1467}
1468
1469static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1470{
1471 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1472 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1473 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1474 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1475
1476 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1477}
1478
1479static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1480{
1481 /* Calculate the display mode Average Bandwidth
1482 * DisplayMode should contain the source and destination dimensions,
1483 * timing, etc.
1484 */
1485 fixed20_12 bpp;
1486 fixed20_12 line_time;
1487 fixed20_12 src_width;
1488 fixed20_12 bandwidth;
1489 fixed20_12 a;
1490
1491 a.full = dfixed_const(1000);
1492 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1493 line_time.full = dfixed_div(line_time, a);
1494 bpp.full = dfixed_const(wm->bytes_per_pixel);
1495 src_width.full = dfixed_const(wm->src_width);
1496 bandwidth.full = dfixed_mul(src_width, bpp);
1497 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1498 bandwidth.full = dfixed_div(bandwidth, line_time);
1499
1500 return dfixed_trunc(bandwidth);
1501}
1502
1503static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1504{
1505 /* First calcualte the latency in ns */
1506 u32 mc_latency = 2000; /* 2000 ns. */
1507 u32 available_bandwidth = dce6_available_bandwidth(wm);
1508 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1509 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1510 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1511 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1512 (wm->num_heads * cursor_line_pair_return_time);
1513 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1514 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1515 u32 tmp, dmif_size = 12288;
1516 fixed20_12 a, b, c;
1517
1518 if (wm->num_heads == 0)
1519 return 0;
1520
1521 a.full = dfixed_const(2);
1522 b.full = dfixed_const(1);
1523 if ((wm->vsc.full > a.full) ||
1524 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1525 (wm->vtaps >= 5) ||
1526 ((wm->vsc.full >= a.full) && wm->interlaced))
1527 max_src_lines_per_dst_line = 4;
1528 else
1529 max_src_lines_per_dst_line = 2;
1530
1531 a.full = dfixed_const(available_bandwidth);
1532 b.full = dfixed_const(wm->num_heads);
1533 a.full = dfixed_div(a, b);
1534
1535 b.full = dfixed_const(mc_latency + 512);
1536 c.full = dfixed_const(wm->disp_clk);
1537 b.full = dfixed_div(b, c);
1538
1539 c.full = dfixed_const(dmif_size);
1540 b.full = dfixed_div(c, b);
1541
1542 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1543
1544 b.full = dfixed_const(1000);
1545 c.full = dfixed_const(wm->disp_clk);
1546 b.full = dfixed_div(c, b);
1547 c.full = dfixed_const(wm->bytes_per_pixel);
1548 b.full = dfixed_mul(b, c);
1549
1550 lb_fill_bw = min(tmp, dfixed_trunc(b));
1551
1552 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1553 b.full = dfixed_const(1000);
1554 c.full = dfixed_const(lb_fill_bw);
1555 b.full = dfixed_div(c, b);
1556 a.full = dfixed_div(a, b);
1557 line_fill_time = dfixed_trunc(a);
1558
1559 if (line_fill_time < wm->active_time)
1560 return latency;
1561 else
1562 return latency + (line_fill_time - wm->active_time);
1563
1564}
1565
1566static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1567{
1568 if (dce6_average_bandwidth(wm) <=
1569 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1570 return true;
1571 else
1572 return false;
1573};
1574
1575static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
1576{
1577 if (dce6_average_bandwidth(wm) <=
1578 (dce6_available_bandwidth(wm) / wm->num_heads))
1579 return true;
1580 else
1581 return false;
1582};
1583
1584static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
1585{
1586 u32 lb_partitions = wm->lb_size / wm->src_width;
1587 u32 line_time = wm->active_time + wm->blank_time;
1588 u32 latency_tolerant_lines;
1589 u32 latency_hiding;
1590 fixed20_12 a;
1591
1592 a.full = dfixed_const(1);
1593 if (wm->vsc.full > a.full)
1594 latency_tolerant_lines = 1;
1595 else {
1596 if (lb_partitions <= (wm->vtaps + 1))
1597 latency_tolerant_lines = 1;
1598 else
1599 latency_tolerant_lines = 2;
1600 }
1601
1602 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
1603
1604 if (dce6_latency_watermark(wm) <= latency_hiding)
1605 return true;
1606 else
1607 return false;
1608}
1609
1610static void dce6_program_watermarks(struct radeon_device *rdev,
1611 struct radeon_crtc *radeon_crtc,
1612 u32 lb_size, u32 num_heads)
1613{
1614 struct drm_display_mode *mode = &radeon_crtc->base.mode;
1615 struct dce6_wm_params wm;
1616 u32 pixel_period;
1617 u32 line_time = 0;
1618 u32 latency_watermark_a = 0, latency_watermark_b = 0;
1619 u32 priority_a_mark = 0, priority_b_mark = 0;
1620 u32 priority_a_cnt = PRIORITY_OFF;
1621 u32 priority_b_cnt = PRIORITY_OFF;
1622 u32 tmp, arb_control3;
1623 fixed20_12 a, b, c;
1624
1625 if (radeon_crtc->base.enabled && num_heads && mode) {
1626 pixel_period = 1000000 / (u32)mode->clock;
1627 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
1628 priority_a_cnt = 0;
1629 priority_b_cnt = 0;
1630
1631 wm.yclk = rdev->pm.current_mclk * 10;
1632 wm.sclk = rdev->pm.current_sclk * 10;
1633 wm.disp_clk = mode->clock;
1634 wm.src_width = mode->crtc_hdisplay;
1635 wm.active_time = mode->crtc_hdisplay * pixel_period;
1636 wm.blank_time = line_time - wm.active_time;
1637 wm.interlaced = false;
1638 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1639 wm.interlaced = true;
1640 wm.vsc = radeon_crtc->vsc;
1641 wm.vtaps = 1;
1642 if (radeon_crtc->rmx_type != RMX_OFF)
1643 wm.vtaps = 2;
1644 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
1645 wm.lb_size = lb_size;
ca7db22b
AD		 1646 if (rdev->family == CHIP_ARUBA)
1647 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
1648 else
1649 wm.dram_channels = si_get_number_of_dram_channels(rdev);
43b3cd99
AD		 1650 wm.num_heads = num_heads;
1651
1652 /* set for high clocks */
1653 latency_watermark_a = min(dce6_latency_watermark(&wm), (u32)65535);
1654 /* set for low clocks */
1655 /* wm.yclk = low clk; wm.sclk = low clk */
1656 latency_watermark_b = min(dce6_latency_watermark(&wm), (u32)65535);
1657
1658 /* possibly force display priority to high */
1659 /* should really do this at mode validation time... */
1660 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
1661 !dce6_average_bandwidth_vs_available_bandwidth(&wm) ||
1662 !dce6_check_latency_hiding(&wm) ||
1663 (rdev->disp_priority == 2)) {
1664 DRM_DEBUG_KMS("force priority to high\n");
1665 priority_a_cnt |= PRIORITY_ALWAYS_ON;
1666 priority_b_cnt |= PRIORITY_ALWAYS_ON;
1667 }
1668
1669 a.full = dfixed_const(1000);
1670 b.full = dfixed_const(mode->clock);
1671 b.full = dfixed_div(b, a);
1672 c.full = dfixed_const(latency_watermark_a);
1673 c.full = dfixed_mul(c, b);
1674 c.full = dfixed_mul(c, radeon_crtc->hsc);
1675 c.full = dfixed_div(c, a);
1676 a.full = dfixed_const(16);
1677 c.full = dfixed_div(c, a);
1678 priority_a_mark = dfixed_trunc(c);
1679 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
1680
1681 a.full = dfixed_const(1000);
1682 b.full = dfixed_const(mode->clock);
1683 b.full = dfixed_div(b, a);
1684 c.full = dfixed_const(latency_watermark_b);
1685 c.full = dfixed_mul(c, b);
1686 c.full = dfixed_mul(c, radeon_crtc->hsc);
1687 c.full = dfixed_div(c, a);
1688 a.full = dfixed_const(16);
1689 c.full = dfixed_div(c, a);
1690 priority_b_mark = dfixed_trunc(c);
1691 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
1692 }
1693
1694 /* select wm A */
1695 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1696 tmp = arb_control3;
1697 tmp &= ~LATENCY_WATERMARK_MASK(3);
1698 tmp |= LATENCY_WATERMARK_MASK(1);
1699 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1700 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1701 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
1702 LATENCY_HIGH_WATERMARK(line_time)));
1703 /* select wm B */
1704 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1705 tmp &= ~LATENCY_WATERMARK_MASK(3);
1706 tmp |= LATENCY_WATERMARK_MASK(2);
1707 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1708 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1709 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
1710 LATENCY_HIGH_WATERMARK(line_time)));
1711 /* restore original selection */
1712 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
1713
1714 /* write the priority marks */
1715 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
1716 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
1717
1718}
1719
1720void dce6_bandwidth_update(struct radeon_device *rdev)
1721{
1722 struct drm_display_mode *mode0 = NULL;
1723 struct drm_display_mode *mode1 = NULL;
1724 u32 num_heads = 0, lb_size;
1725 int i;
1726
1727 radeon_update_display_priority(rdev);
1728
1729 for (i = 0; i < rdev->num_crtc; i++) {
1730 if (rdev->mode_info.crtcs[i]->base.enabled)
1731 num_heads++;
1732 }
1733 for (i = 0; i < rdev->num_crtc; i += 2) {
1734 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
1735 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
1736 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
1737 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
1738 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
1739 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
1740 }
1741}
1742
0a96d72b
AD		 1743/*
1744 * Core functions
1745 */
0a96d72b
AD		 1746static void si_tiling_mode_table_init(struct radeon_device *rdev)
1747{
1748 const u32 num_tile_mode_states = 32;
1749 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1750
1751 switch (rdev->config.si.mem_row_size_in_kb) {
1752 case 1:
1753 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1754 break;
1755 case 2:
1756 default:
1757 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1758 break;
1759 case 4:
1760 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1761 break;
1762 }
1763
1764 if ((rdev->family == CHIP_TAHITI) ||
1765 (rdev->family == CHIP_PITCAIRN)) {
1766 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1767 switch (reg_offset) {
1768 case 0: /* non-AA compressed depth or any compressed stencil */
1769 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1770 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1771 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1773 NUM_BANKS(ADDR_SURF_16_BANK) |
1774 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1775 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1776 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1777 break;
1778 case 1: /* 2xAA/4xAA compressed depth only */
1779 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1780 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1781 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1782 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1783 NUM_BANKS(ADDR_SURF_16_BANK) |
1784 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1785 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1786 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1787 break;
1788 case 2: /* 8xAA compressed depth only */
1789 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1790 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1791 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1792 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1793 NUM_BANKS(ADDR_SURF_16_BANK) |
1794 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1795 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1796 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1797 break;
1798 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
1799 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1800 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1801 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1802 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1803 NUM_BANKS(ADDR_SURF_16_BANK) |
1804 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1805 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1806 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1807 break;
1808 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
1809 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1810 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1811 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1812 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1813 NUM_BANKS(ADDR_SURF_16_BANK) |
1814 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1815 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1816 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1817 break;
1818 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
1819 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1820 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1821 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1822 TILE_SPLIT(split_equal_to_row_size) |
1823 NUM_BANKS(ADDR_SURF_16_BANK) |
1824 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1825 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1826 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1827 break;
1828 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
1829 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1830 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1831 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1832 TILE_SPLIT(split_equal_to_row_size) |
1833 NUM_BANKS(ADDR_SURF_16_BANK) |
1834 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1835 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1836 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1837 break;
1838 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
1839 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1840 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1841 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1842 TILE_SPLIT(split_equal_to_row_size) |
1843 NUM_BANKS(ADDR_SURF_16_BANK) |
1844 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1845 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1846 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1847 break;
1848 case 8: /* 1D and 1D Array Surfaces */
1849 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1850 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1851 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1852 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1853 NUM_BANKS(ADDR_SURF_16_BANK) |
1854 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1855 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1856 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1857 break;
1858 case 9: /* Displayable maps. */
1859 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1860 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1861 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1862 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1863 NUM_BANKS(ADDR_SURF_16_BANK) |
1864 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1865 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1866 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1867 break;
1868 case 10: /* Display 8bpp. */
1869 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1870 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1871 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1872 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1873 NUM_BANKS(ADDR_SURF_16_BANK) |
1874 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1875 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1876 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1877 break;
1878 case 11: /* Display 16bpp. */
1879 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1880 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1881 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1882 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1883 NUM_BANKS(ADDR_SURF_16_BANK) |
1884 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1885 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1886 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1887 break;
1888 case 12: /* Display 32bpp. */
1889 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1890 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1891 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1892 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1893 NUM_BANKS(ADDR_SURF_16_BANK) |
1894 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1895 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1896 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1897 break;
1898 case 13: /* Thin. */
1899 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1900 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1901 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1902 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1903 NUM_BANKS(ADDR_SURF_16_BANK) |
1904 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1905 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1906 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1907 break;
1908 case 14: /* Thin 8 bpp. */
1909 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1910 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1911 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1912 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1913 NUM_BANKS(ADDR_SURF_16_BANK) |
1914 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1917 break;
1918 case 15: /* Thin 16 bpp. */
1919 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1920 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1921 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1922 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1923 NUM_BANKS(ADDR_SURF_16_BANK) |
1924 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1925 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1926 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1927 break;
1928 case 16: /* Thin 32 bpp. */
1929 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1930 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1931 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1932 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1933 NUM_BANKS(ADDR_SURF_16_BANK) |
1934 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1935 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1936 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1937 break;
1938 case 17: /* Thin 64 bpp. */
1939 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1940 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1941 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1942 TILE_SPLIT(split_equal_to_row_size) |
1943 NUM_BANKS(ADDR_SURF_16_BANK) |
1944 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1945 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1946 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1947 break;
1948 case 21: /* 8 bpp PRT. */
1949 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1950 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1951 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1952 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1953 NUM_BANKS(ADDR_SURF_16_BANK) |
1954 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1957 break;
1958 case 22: /* 16 bpp PRT */
1959 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1960 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1961 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1962 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1963 NUM_BANKS(ADDR_SURF_16_BANK) |
1964 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1965 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1966 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
1967 break;
1968 case 23: /* 32 bpp PRT */
1969 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1970 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1971 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1972 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1973 NUM_BANKS(ADDR_SURF_16_BANK) |
1974 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1975 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1976 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1977 break;
1978 case 24: /* 64 bpp PRT */
1979 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1980 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1981 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1982 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1983 NUM_BANKS(ADDR_SURF_16_BANK) |
1984 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1987 break;
1988 case 25: /* 128 bpp PRT */
1989 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1990 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1991 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1992 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1993 NUM_BANKS(ADDR_SURF_8_BANK) |
1994 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1995 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1996 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1997 break;
1998 default:
1999 gb_tile_moden = 0;
2000 break;
2001 }
64d7b8be 2002 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2003 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2004 }
d0ae7fcc 2005 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2006 (rdev->family == CHIP_OLAND) ||
2007 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2008 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2009 switch (reg_offset) {
2010 case 0: /* non-AA compressed depth or any compressed stencil */
2011 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2012 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2013 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2014 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2015 NUM_BANKS(ADDR_SURF_16_BANK) |
2016 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2017 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2018 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2019 break;
2020 case 1: /* 2xAA/4xAA compressed depth only */
2021 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2022 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2023 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2024 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2025 NUM_BANKS(ADDR_SURF_16_BANK) |
2026 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2027 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2028 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2029 break;
2030 case 2: /* 8xAA compressed depth only */
2031 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2032 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2033 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2034 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2035 NUM_BANKS(ADDR_SURF_16_BANK) |
2036 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2037 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2038 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2039 break;
2040 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2041 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2042 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2043 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2044 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2045 NUM_BANKS(ADDR_SURF_16_BANK) |
2046 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2047 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2048 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2049 break;
2050 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2051 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2052 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2053 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2054 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2055 NUM_BANKS(ADDR_SURF_16_BANK) |
2056 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2057 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2058 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2059 break;
2060 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2061 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2062 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2063 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2064 TILE_SPLIT(split_equal_to_row_size) |
2065 NUM_BANKS(ADDR_SURF_16_BANK) |
2066 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2067 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2068 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2069 break;
2070 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2071 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2072 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2073 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2074 TILE_SPLIT(split_equal_to_row_size) |
2075 NUM_BANKS(ADDR_SURF_16_BANK) |
2076 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2077 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2078 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2079 break;
2080 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2081 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2082 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2083 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2084 TILE_SPLIT(split_equal_to_row_size) |
2085 NUM_BANKS(ADDR_SURF_16_BANK) |
2086 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2087 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2088 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2089 break;
2090 case 8: /* 1D and 1D Array Surfaces */
2091 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2092 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2093 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2094 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2095 NUM_BANKS(ADDR_SURF_16_BANK) |
2096 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2097 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2098 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2099 break;
2100 case 9: /* Displayable maps. */
2101 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2102 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2103 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2104 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2105 NUM_BANKS(ADDR_SURF_16_BANK) |
2106 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2107 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2108 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2109 break;
2110 case 10: /* Display 8bpp. */
2111 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2112 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2113 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2114 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2115 NUM_BANKS(ADDR_SURF_16_BANK) |
2116 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2117 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2118 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2119 break;
2120 case 11: /* Display 16bpp. */
2121 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2122 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2123 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2124 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2125 NUM_BANKS(ADDR_SURF_16_BANK) |
2126 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2127 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2128 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2129 break;
2130 case 12: /* Display 32bpp. */
2131 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2132 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2133 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2134 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2135 NUM_BANKS(ADDR_SURF_16_BANK) |
2136 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2137 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2138 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2139 break;
2140 case 13: /* Thin. */
2141 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2142 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2143 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2144 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2145 NUM_BANKS(ADDR_SURF_16_BANK) |
2146 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2147 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2148 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2149 break;
2150 case 14: /* Thin 8 bpp. */
2151 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2152 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2153 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2154 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2155 NUM_BANKS(ADDR_SURF_16_BANK) |
2156 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2157 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2158 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2159 break;
2160 case 15: /* Thin 16 bpp. */
2161 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2162 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2163 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2164 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2165 NUM_BANKS(ADDR_SURF_16_BANK) |
2166 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2167 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2168 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2169 break;
2170 case 16: /* Thin 32 bpp. */
2171 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2172 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2173 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2174 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2175 NUM_BANKS(ADDR_SURF_16_BANK) |
2176 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2177 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2178 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2179 break;
2180 case 17: /* Thin 64 bpp. */
2181 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2182 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2183 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2184 TILE_SPLIT(split_equal_to_row_size) |
2185 NUM_BANKS(ADDR_SURF_16_BANK) |
2186 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2187 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2188 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2189 break;
2190 case 21: /* 8 bpp PRT. */
2191 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2192 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2193 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2194 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2195 NUM_BANKS(ADDR_SURF_16_BANK) |
2196 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2197 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2198 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2199 break;
2200 case 22: /* 16 bpp PRT */
2201 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2202 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2203 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2204 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2205 NUM_BANKS(ADDR_SURF_16_BANK) |
2206 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2207 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2208 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2209 break;
2210 case 23: /* 32 bpp PRT */
2211 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2212 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2213 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2214 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2215 NUM_BANKS(ADDR_SURF_16_BANK) |
2216 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2217 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2218 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2219 break;
2220 case 24: /* 64 bpp PRT */
2221 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2222 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2223 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2224 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2225 NUM_BANKS(ADDR_SURF_16_BANK) |
2226 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2227 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2228 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2229 break;
2230 case 25: /* 128 bpp PRT */
2231 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2232 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2233 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2234 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2235 NUM_BANKS(ADDR_SURF_8_BANK) |
2236 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2237 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2238 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2239 break;
2240 default:
2241 gb_tile_moden = 0;
2242 break;
2243 }
64d7b8be 2244 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2245 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2246 }
2247 } else
2248 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2249}
2250
1a8ca750
AD		 2251static void si_select_se_sh(struct radeon_device *rdev,
2252 u32 se_num, u32 sh_num)
2253{
2254 u32 data = INSTANCE_BROADCAST_WRITES;
2255
2256 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2257 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2258 else if (se_num == 0xffffffff)
2259 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2260 else if (sh_num == 0xffffffff)
2261 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2262 else
2263 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2264 WREG32(GRBM_GFX_INDEX, data);
2265}
2266
2267static u32 si_create_bitmask(u32 bit_width)
2268{
2269 u32 i, mask = 0;
2270
2271 for (i = 0; i < bit_width; i++) {
2272 mask <<= 1;
2273 mask |= 1;
2274 }
2275 return mask;
2276}
2277
2278static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2279{
2280 u32 data, mask;
2281
2282 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2283 if (data & 1)
2284 data &= INACTIVE_CUS_MASK;
2285 else
2286 data = 0;
2287 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2288
2289 data >>= INACTIVE_CUS_SHIFT;
2290
2291 mask = si_create_bitmask(cu_per_sh);
2292
2293 return ~data & mask;
2294}
2295
2296static void si_setup_spi(struct radeon_device *rdev,
2297 u32 se_num, u32 sh_per_se,
2298 u32 cu_per_sh)
2299{
2300 int i, j, k;
2301 u32 data, mask, active_cu;
2302
2303 for (i = 0; i < se_num; i++) {
2304 for (j = 0; j < sh_per_se; j++) {
2305 si_select_se_sh(rdev, i, j);
2306 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2307 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2308
2309 mask = 1;
2310 for (k = 0; k < 16; k++) {
2311 mask <<= k;
2312 if (active_cu & mask) {
2313 data &= ~mask;
2314 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2315 break;
2316 }
2317 }
2318 }
2319 }
2320 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2321}
2322
2323static u32 si_get_rb_disabled(struct radeon_device *rdev,
2324 u32 max_rb_num, u32 se_num,
2325 u32 sh_per_se)
2326{
2327 u32 data, mask;
2328
2329 data = RREG32(CC_RB_BACKEND_DISABLE);
2330 if (data & 1)
2331 data &= BACKEND_DISABLE_MASK;
2332 else
2333 data = 0;
2334 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2335
2336 data >>= BACKEND_DISABLE_SHIFT;
2337
2338 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2339
2340 return data & mask;
2341}
2342
2343static void si_setup_rb(struct radeon_device *rdev,
2344 u32 se_num, u32 sh_per_se,
2345 u32 max_rb_num)
2346{
2347 int i, j;
2348 u32 data, mask;
2349 u32 disabled_rbs = 0;
2350 u32 enabled_rbs = 0;
2351
2352 for (i = 0; i < se_num; i++) {
2353 for (j = 0; j < sh_per_se; j++) {
2354 si_select_se_sh(rdev, i, j);
2355 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2356 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2357 }
2358 }
2359 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2360
2361 mask = 1;
2362 for (i = 0; i < max_rb_num; i++) {
2363 if (!(disabled_rbs & mask))
2364 enabled_rbs |= mask;
2365 mask <<= 1;
2366 }
2367
2368 for (i = 0; i < se_num; i++) {
2369 si_select_se_sh(rdev, i, 0xffffffff);
2370 data = 0;
2371 for (j = 0; j < sh_per_se; j++) {
2372 switch (enabled_rbs & 3) {
2373 case 1:
2374 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2375 break;
2376 case 2:
2377 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2378 break;
2379 case 3:
2380 default:
2381 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2382 break;
2383 }
2384 enabled_rbs >>= 2;
2385 }
2386 WREG32(PA_SC_RASTER_CONFIG, data);
2387 }
2388 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2389}
2390
0a96d72b
AD		 2391static void si_gpu_init(struct radeon_device *rdev)
2392{
0a96d72b
AD		 2393 u32 gb_addr_config = 0;
2394 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2395 u32 sx_debug_1;
0a96d72b
AD		 2396 u32 hdp_host_path_cntl;
2397 u32 tmp;
2398 int i, j;
2399
2400 switch (rdev->family) {
2401 case CHIP_TAHITI:
2402 rdev->config.si.max_shader_engines = 2;
0a96d72b 2403 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2404 rdev->config.si.max_cu_per_sh = 8;
2405 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2406 rdev->config.si.max_backends_per_se = 4;
2407 rdev->config.si.max_texture_channel_caches = 12;
2408 rdev->config.si.max_gprs = 256;
2409 rdev->config.si.max_gs_threads = 32;
2410 rdev->config.si.max_hw_contexts = 8;
2411
2412 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2413 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2414 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2415 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2416 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2417 break;
2418 case CHIP_PITCAIRN:
2419 rdev->config.si.max_shader_engines = 2;
0a96d72b 2420 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2421 rdev->config.si.max_cu_per_sh = 5;
2422 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2423 rdev->config.si.max_backends_per_se = 4;
2424 rdev->config.si.max_texture_channel_caches = 8;
2425 rdev->config.si.max_gprs = 256;
2426 rdev->config.si.max_gs_threads = 32;
2427 rdev->config.si.max_hw_contexts = 8;
2428
2429 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2430 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2431 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2432 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2433 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2434 break;
2435 case CHIP_VERDE:
2436 default:
2437 rdev->config.si.max_shader_engines = 1;
0a96d72b 2438 rdev->config.si.max_tile_pipes = 4;
1a8ca750
AD		 2439 rdev->config.si.max_cu_per_sh = 2;
2440 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2441 rdev->config.si.max_backends_per_se = 4;
2442 rdev->config.si.max_texture_channel_caches = 4;
2443 rdev->config.si.max_gprs = 256;
2444 rdev->config.si.max_gs_threads = 32;
2445 rdev->config.si.max_hw_contexts = 8;
2446
d0ae7fcc
AD		 2447 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2448 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2449 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2450 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2451 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2452 break;
2453 case CHIP_OLAND:
2454 rdev->config.si.max_shader_engines = 1;
2455 rdev->config.si.max_tile_pipes = 4;
2456 rdev->config.si.max_cu_per_sh = 6;
2457 rdev->config.si.max_sh_per_se = 1;
2458 rdev->config.si.max_backends_per_se = 2;
2459 rdev->config.si.max_texture_channel_caches = 4;
2460 rdev->config.si.max_gprs = 256;
2461 rdev->config.si.max_gs_threads = 16;
2462 rdev->config.si.max_hw_contexts = 8;
2463
0a96d72b
AD		 2464 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2465 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2466 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2467 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2468 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2469 break;
8b02859d
AD		 2470 case CHIP_HAINAN:
2471 rdev->config.si.max_shader_engines = 1;
2472 rdev->config.si.max_tile_pipes = 4;
2473 rdev->config.si.max_cu_per_sh = 5;
2474 rdev->config.si.max_sh_per_se = 1;
2475 rdev->config.si.max_backends_per_se = 1;
2476 rdev->config.si.max_texture_channel_caches = 2;
2477 rdev->config.si.max_gprs = 256;
2478 rdev->config.si.max_gs_threads = 16;
2479 rdev->config.si.max_hw_contexts = 8;
2480
2481 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2482 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2483 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2484 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2485 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2486 break;
0a96d72b
AD		 2487 }
2488
2489 /* Initialize HDP */
2490 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2491 WREG32((0x2c14 + j), 0x00000000);
2492 WREG32((0x2c18 + j), 0x00000000);
2493 WREG32((0x2c1c + j), 0x00000000);
2494 WREG32((0x2c20 + j), 0x00000000);
2495 WREG32((0x2c24 + j), 0x00000000);
2496 }
2497
2498 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2499
2500 evergreen_fix_pci_max_read_req_size(rdev);
2501
2502 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2503
2504 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2505 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2506
0a96d72b 2507 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 2508 rdev->config.si.mem_max_burst_length_bytes = 256;
2509 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2510 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2511 if (rdev->config.si.mem_row_size_in_kb > 4)
2512 rdev->config.si.mem_row_size_in_kb = 4;
2513 /* XXX use MC settings? */
2514 rdev->config.si.shader_engine_tile_size = 32;
2515 rdev->config.si.num_gpus = 1;
2516 rdev->config.si.multi_gpu_tile_size = 64;
2517
1a8ca750
AD		 2518 /* fix up row size */
2519 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 2520 switch (rdev->config.si.mem_row_size_in_kb) {
2521 case 1:
2522 default:
2523 gb_addr_config |= ROW_SIZE(0);
2524 break;
2525 case 2:
2526 gb_addr_config |= ROW_SIZE(1);
2527 break;
2528 case 4:
2529 gb_addr_config |= ROW_SIZE(2);
2530 break;
2531 }
2532
0a96d72b
AD		 2533 /* setup tiling info dword. gb_addr_config is not adequate since it does
2534 * not have bank info, so create a custom tiling dword.
2535 * bits 3:0 num_pipes
2536 * bits 7:4 num_banks
2537 * bits 11:8 group_size
2538 * bits 15:12 row_size
2539 */
2540 rdev->config.si.tile_config = 0;
2541 switch (rdev->config.si.num_tile_pipes) {
2542 case 1:
2543 rdev->config.si.tile_config |= (0 << 0);
2544 break;
2545 case 2:
2546 rdev->config.si.tile_config |= (1 << 0);
2547 break;
2548 case 4:
2549 rdev->config.si.tile_config |= (2 << 0);
2550 break;
2551 case 8:
2552 default:
2553 /* XXX what about 12? */
2554 rdev->config.si.tile_config |= (3 << 0);
2555 break;
dca571a6
CK		 2556 }
2557 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
2558 case 0: /* four banks */
1a8ca750 2559 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 2560 break;
2561 case 1: /* eight banks */
2562 rdev->config.si.tile_config |= 1 << 4;
2563 break;
2564 case 2: /* sixteen banks */
2565 default:
2566 rdev->config.si.tile_config |= 2 << 4;
2567 break;
2568 }
0a96d72b
AD		 2569 rdev->config.si.tile_config |=
2570 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2571 rdev->config.si.tile_config |=
2572 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2573
0a96d72b
AD		 2574 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2575 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 2576 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 2577 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 2578 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
2579 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
9a21059d
CK		 2580 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2581 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2582 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
0a96d72b 2583
1a8ca750 2584 si_tiling_mode_table_init(rdev);
0a96d72b 2585
1a8ca750
AD		 2586 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
2587 rdev->config.si.max_sh_per_se,
2588 rdev->config.si.max_backends_per_se);
0a96d72b 2589
1a8ca750
AD		 2590 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
2591 rdev->config.si.max_sh_per_se,
2592 rdev->config.si.max_cu_per_sh);
0a96d72b 2593
0a96d72b
AD		 2594
2595 /* set HW defaults for 3D engine */
2596 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2597 ROQ_IB2_START(0x2b)));
2598 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2599
2600 sx_debug_1 = RREG32(SX_DEBUG_1);
2601 WREG32(SX_DEBUG_1, sx_debug_1);
2602
2603 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2604
2605 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
2606 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
2607 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
2608 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
2609
2610 WREG32(VGT_NUM_INSTANCES, 1);
2611
2612 WREG32(CP_PERFMON_CNTL, 0);
2613
2614 WREG32(SQ_CONFIG, 0);
2615
2616 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2617 FORCE_EOV_MAX_REZ_CNT(255)));
2618
2619 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2620 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2621
2622 WREG32(VGT_GS_VERTEX_REUSE, 16);
2623 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2624
2625 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
2626 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
2627 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
2628 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
2629 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
2630 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
2631 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
2632 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
2633
2634 tmp = RREG32(HDP_MISC_CNTL);
2635 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2636 WREG32(HDP_MISC_CNTL, tmp);
2637
2638 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2639 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2640
2641 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2642
2643 udelay(50);
2644}
c476dde2 2645
2ece2e8b
AD		 2646/*
2647 * GPU scratch registers helpers function.
2648 */
2649static void si_scratch_init(struct radeon_device *rdev)
2650{
2651 int i;
2652
2653 rdev->scratch.num_reg = 7;
2654 rdev->scratch.reg_base = SCRATCH_REG0;
2655 for (i = 0; i < rdev->scratch.num_reg; i++) {
2656 rdev->scratch.free[i] = true;
2657 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2658 }
2659}
2660
2661void si_fence_ring_emit(struct radeon_device *rdev,
2662 struct radeon_fence *fence)
2663{
2664 struct radeon_ring *ring = &rdev->ring[fence->ring];
2665 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2666
2667 /* flush read cache over gart */
2668 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2669 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
2670 radeon_ring_write(ring, 0);
2671 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2672 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2673 PACKET3_TC_ACTION_ENA |
2674 PACKET3_SH_KCACHE_ACTION_ENA |
2675 PACKET3_SH_ICACHE_ACTION_ENA);
2676 radeon_ring_write(ring, 0xFFFFFFFF);
2677 radeon_ring_write(ring, 0);
2678 radeon_ring_write(ring, 10); /* poll interval */
2679 /* EVENT_WRITE_EOP - flush caches, send int */
2680 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2681 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
2682 radeon_ring_write(ring, addr & 0xffffffff);
2683 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2684 radeon_ring_write(ring, fence->seq);
2685 radeon_ring_write(ring, 0);
2686}
2687
2688/*
2689 * IB stuff
2690 */
2691void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2692{
876dc9f3 2693 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 2694 u32 header;
2695
a85a7da4
AD		 2696 if (ib->is_const_ib) {
2697 /* set switch buffer packet before const IB */
2698 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2699 radeon_ring_write(ring, 0);
45df6803 2700
2ece2e8b 2701 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 2702 } else {
89d35807 2703 u32 next_rptr;
a85a7da4 2704 if (ring->rptr_save_reg) {
89d35807 2705 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 2706 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2707 radeon_ring_write(ring, ((ring->rptr_save_reg -
2708 PACKET3_SET_CONFIG_REG_START) >> 2));
2709 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2710 } else if (rdev->wb.enabled) {
2711 next_rptr = ring->wptr + 5 + 4 + 8;
2712 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2713 radeon_ring_write(ring, (1 << 8));
2714 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2715 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
2716 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 2717 }
2718
2ece2e8b 2719 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 2720 }
2ece2e8b
AD		 2721
2722 radeon_ring_write(ring, header);
2723 radeon_ring_write(ring,
2724#ifdef __BIG_ENDIAN
2725 (2 << 0) |
2726#endif
2727 (ib->gpu_addr & 0xFFFFFFFC));
2728 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 2729 radeon_ring_write(ring, ib->length_dw |
2730 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 2731
a85a7da4
AD		 2732 if (!ib->is_const_ib) {
2733 /* flush read cache over gart for this vmid */
2734 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2735 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 2736 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 2737 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2738 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2739 PACKET3_TC_ACTION_ENA |
2740 PACKET3_SH_KCACHE_ACTION_ENA |
2741 PACKET3_SH_ICACHE_ACTION_ENA);
2742 radeon_ring_write(ring, 0xFFFFFFFF);
2743 radeon_ring_write(ring, 0);
2744 radeon_ring_write(ring, 10); /* poll interval */
2745 }
2ece2e8b
AD		 2746}
2747
48c0c902
AD		 2748/*
2749 * CP.
2750 */
2751static void si_cp_enable(struct radeon_device *rdev, bool enable)
2752{
2753 if (enable)
2754 WREG32(CP_ME_CNTL, 0);
2755 else {
2756 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2757 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
2758 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 2759 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2760 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2761 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 2762 }
2763 udelay(50);
2764}
2765
2766static int si_cp_load_microcode(struct radeon_device *rdev)
2767{
2768 const __be32 *fw_data;
2769 int i;
2770
2771 if (!rdev->me_fw || !rdev->pfp_fw)
2772 return -EINVAL;
2773
2774 si_cp_enable(rdev, false);
2775
2776 /* PFP */
2777 fw_data = (const __be32 *)rdev->pfp_fw->data;
2778 WREG32(CP_PFP_UCODE_ADDR, 0);
2779 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
2780 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2781 WREG32(CP_PFP_UCODE_ADDR, 0);
2782
2783 /* CE */
2784 fw_data = (const __be32 *)rdev->ce_fw->data;
2785 WREG32(CP_CE_UCODE_ADDR, 0);
2786 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
2787 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
2788 WREG32(CP_CE_UCODE_ADDR, 0);
2789
2790 /* ME */
2791 fw_data = (const __be32 *)rdev->me_fw->data;
2792 WREG32(CP_ME_RAM_WADDR, 0);
2793 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
2794 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2795 WREG32(CP_ME_RAM_WADDR, 0);
2796
2797 WREG32(CP_PFP_UCODE_ADDR, 0);
2798 WREG32(CP_CE_UCODE_ADDR, 0);
2799 WREG32(CP_ME_RAM_WADDR, 0);
2800 WREG32(CP_ME_RAM_RADDR, 0);
2801 return 0;
2802}
2803
2804static int si_cp_start(struct radeon_device *rdev)
2805{
2806 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2807 int r, i;
2808
2809 r = radeon_ring_lock(rdev, ring, 7 + 4);
2810 if (r) {
2811 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2812 return r;
2813 }
2814 /* init the CP */
2815 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2816 radeon_ring_write(ring, 0x1);
2817 radeon_ring_write(ring, 0x0);
2818 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
2819 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2820 radeon_ring_write(ring, 0);
2821 radeon_ring_write(ring, 0);
2822
2823 /* init the CE partitions */
2824 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2825 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2826 radeon_ring_write(ring, 0xc000);
2827 radeon_ring_write(ring, 0xe000);
2828 radeon_ring_unlock_commit(rdev, ring);
2829
2830 si_cp_enable(rdev, true);
2831
2832 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
2833 if (r) {
2834 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2835 return r;
2836 }
2837
2838 /* setup clear context state */
2839 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2840 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2841
2842 for (i = 0; i < si_default_size; i++)
2843 radeon_ring_write(ring, si_default_state[i]);
2844
2845 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2846 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2847
2848 /* set clear context state */
2849 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2850 radeon_ring_write(ring, 0);
2851
2852 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2853 radeon_ring_write(ring, 0x00000316);
2854 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2855 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
2856
2857 radeon_ring_unlock_commit(rdev, ring);
2858
2859 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
2860 ring = &rdev->ring[i];
2861 r = radeon_ring_lock(rdev, ring, 2);
2862
2863 /* clear the compute context state */
2864 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
2865 radeon_ring_write(ring, 0);
2866
2867 radeon_ring_unlock_commit(rdev, ring);
2868 }
2869
2870 return 0;
2871}
2872
2873static void si_cp_fini(struct radeon_device *rdev)
2874{
45df6803 2875 struct radeon_ring *ring;
48c0c902 2876 si_cp_enable(rdev, false);
45df6803
CK		 2877
2878 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2879 radeon_ring_fini(rdev, ring);
2880 radeon_scratch_free(rdev, ring->rptr_save_reg);
2881
2882 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
2883 radeon_ring_fini(rdev, ring);
2884 radeon_scratch_free(rdev, ring->rptr_save_reg);
2885
2886 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
2887 radeon_ring_fini(rdev, ring);
2888 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 2889}
2890
2891static int si_cp_resume(struct radeon_device *rdev)
2892{
2893 struct radeon_ring *ring;
2894 u32 tmp;
2895 u32 rb_bufsz;
2896 int r;
2897
2898 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2899 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2900 SOFT_RESET_PA |
2901 SOFT_RESET_VGT |
2902 SOFT_RESET_SPI |
2903 SOFT_RESET_SX));
2904 RREG32(GRBM_SOFT_RESET);
2905 mdelay(15);
2906 WREG32(GRBM_SOFT_RESET, 0);
2907 RREG32(GRBM_SOFT_RESET);
2908
2909 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2910 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2911
2912 /* Set the write pointer delay */
2913 WREG32(CP_RB_WPTR_DELAY, 0);
2914
2915 WREG32(CP_DEBUG, 0);
2916 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2917
2918 /* ring 0 - compute and gfx */
2919 /* Set ring buffer size */
2920 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2921 rb_bufsz = drm_order(ring->ring_size / 8);
2922 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2923#ifdef __BIG_ENDIAN
2924 tmp |= BUF_SWAP_32BIT;
2925#endif
2926 WREG32(CP_RB0_CNTL, tmp);
2927
2928 /* Initialize the ring buffer's read and write pointers */
2929 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
2930 ring->wptr = 0;
2931 WREG32(CP_RB0_WPTR, ring->wptr);
2932
48fc7f7e 2933 /* set the wb address whether it's enabled or not */
48c0c902
AD		 2934 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
2935 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2936
2937 if (rdev->wb.enabled)
2938 WREG32(SCRATCH_UMSK, 0xff);
2939 else {
2940 tmp |= RB_NO_UPDATE;
2941 WREG32(SCRATCH_UMSK, 0);
2942 }
2943
2944 mdelay(1);
2945 WREG32(CP_RB0_CNTL, tmp);
2946
2947 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
2948
2949 ring->rptr = RREG32(CP_RB0_RPTR);
2950
2951 /* ring1 - compute only */
2952 /* Set ring buffer size */
2953 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
2954 rb_bufsz = drm_order(ring->ring_size / 8);
2955 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2956#ifdef __BIG_ENDIAN
2957 tmp |= BUF_SWAP_32BIT;
2958#endif
2959 WREG32(CP_RB1_CNTL, tmp);
2960
2961 /* Initialize the ring buffer's read and write pointers */
2962 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
2963 ring->wptr = 0;
2964 WREG32(CP_RB1_WPTR, ring->wptr);
2965
48fc7f7e 2966 /* set the wb address whether it's enabled or not */
48c0c902
AD		 2967 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
2968 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
2969
2970 mdelay(1);
2971 WREG32(CP_RB1_CNTL, tmp);
2972
2973 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
2974
2975 ring->rptr = RREG32(CP_RB1_RPTR);
2976
2977 /* ring2 - compute only */
2978 /* Set ring buffer size */
2979 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
2980 rb_bufsz = drm_order(ring->ring_size / 8);
2981 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2982#ifdef __BIG_ENDIAN
2983 tmp |= BUF_SWAP_32BIT;
2984#endif
2985 WREG32(CP_RB2_CNTL, tmp);
2986
2987 /* Initialize the ring buffer's read and write pointers */
2988 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
2989 ring->wptr = 0;
2990 WREG32(CP_RB2_WPTR, ring->wptr);
2991
48fc7f7e 2992 /* set the wb address whether it's enabled or not */
48c0c902
AD		 2993 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
2994 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
2995
2996 mdelay(1);
2997 WREG32(CP_RB2_CNTL, tmp);
2998
2999 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3000
3001 ring->rptr = RREG32(CP_RB2_RPTR);
3002
3003 /* start the rings */
3004 si_cp_start(rdev);
3005 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3006 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3007 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3008 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3009 if (r) {
3010 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3011 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3012 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3013 return r;
3014 }
3015 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3016 if (r) {
3017 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3018 }
3019 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3020 if (r) {
3021 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3022 }
3023
3024 return 0;
3025}
3026
014bb209 3027static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3028{
014bb209 3029 u32 reset_mask = 0;
1c534671 3030 u32 tmp;
06bc6df0 3031
014bb209
AD		 3032 /* GRBM_STATUS */
3033 tmp = RREG32(GRBM_STATUS);
3034 if (tmp & (PA_BUSY | SC_BUSY |
3035 BCI_BUSY | SX_BUSY |
3036 TA_BUSY | VGT_BUSY |
3037 DB_BUSY | CB_BUSY |
3038 GDS_BUSY | SPI_BUSY |
3039 IA_BUSY | IA_BUSY_NO_DMA))
3040 reset_mask |= RADEON_RESET_GFX;
3041
3042 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3043 CP_BUSY | CP_COHERENCY_BUSY))
3044 reset_mask |= RADEON_RESET_CP;
3045
3046 if (tmp & GRBM_EE_BUSY)
3047 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3048
3049 /* GRBM_STATUS2 */
3050 tmp = RREG32(GRBM_STATUS2);
3051 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3052 reset_mask |= RADEON_RESET_RLC;
3053
3054 /* DMA_STATUS_REG 0 */
3055 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3056 if (!(tmp & DMA_IDLE))
3057 reset_mask |= RADEON_RESET_DMA;
3058
3059 /* DMA_STATUS_REG 1 */
3060 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3061 if (!(tmp & DMA_IDLE))
3062 reset_mask |= RADEON_RESET_DMA1;
3063
3064 /* SRBM_STATUS2 */
3065 tmp = RREG32(SRBM_STATUS2);
3066 if (tmp & DMA_BUSY)
3067 reset_mask |= RADEON_RESET_DMA;
3068
3069 if (tmp & DMA1_BUSY)
3070 reset_mask |= RADEON_RESET_DMA1;
3071
3072 /* SRBM_STATUS */
3073 tmp = RREG32(SRBM_STATUS);
3074
3075 if (tmp & IH_BUSY)
3076 reset_mask |= RADEON_RESET_IH;
3077
3078 if (tmp & SEM_BUSY)
3079 reset_mask |= RADEON_RESET_SEM;
3080
3081 if (tmp & GRBM_RQ_PENDING)
3082 reset_mask |= RADEON_RESET_GRBM;
3083
3084 if (tmp & VMC_BUSY)
3085 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3086
014bb209
AD		 3087 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3088 MCC_BUSY | MCD_BUSY))
3089 reset_mask |= RADEON_RESET_MC;
3090
3091 if (evergreen_is_display_hung(rdev))
3092 reset_mask |= RADEON_RESET_DISPLAY;
3093
3094 /* VM_L2_STATUS */
3095 tmp = RREG32(VM_L2_STATUS);
3096 if (tmp & L2_BUSY)
3097 reset_mask |= RADEON_RESET_VMC;
3098
d808fc88
AD		 3099 /* Skip MC reset as it's mostly likely not hung, just busy */
3100 if (reset_mask & RADEON_RESET_MC) {
3101 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3102 reset_mask &= ~RADEON_RESET_MC;
3103 }
3104
014bb209
AD		 3105 return reset_mask;
3106}
3107
3108static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3109{
3110 struct evergreen_mc_save save;
3111 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3112 u32 tmp;
19fc42ed 3113
06bc6df0 3114 if (reset_mask == 0)
014bb209 3115 return;
06bc6df0
AD		 3116
3117 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3118
1c534671 3119 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3120 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3121 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3122 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3123 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3124
1c534671
AD		 3125 /* Disable CP parsing/prefetching */
3126 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3127
3128 if (reset_mask & RADEON_RESET_DMA) {
3129 /* dma0 */
3130 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3131 tmp &= ~DMA_RB_ENABLE;
3132 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3133 }
3134 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3135 /* dma1 */
3136 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3137 tmp &= ~DMA_RB_ENABLE;
3138 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3139 }
3140
f770d78a
AD		 3141 udelay(50);
3142
3143 evergreen_mc_stop(rdev, &save);
3144 if (evergreen_mc_wait_for_idle(rdev)) {
3145 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3146 }
3147
1c534671
AD		 3148 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3149 grbm_soft_reset = SOFT_RESET_CB |
3150 SOFT_RESET_DB |
3151 SOFT_RESET_GDS |
3152 SOFT_RESET_PA |
3153 SOFT_RESET_SC |
3154 SOFT_RESET_BCI |
3155 SOFT_RESET_SPI |
3156 SOFT_RESET_SX |
3157 SOFT_RESET_TC |
3158 SOFT_RESET_TA |
3159 SOFT_RESET_VGT |
3160 SOFT_RESET_IA;
3161 }
3162
3163 if (reset_mask & RADEON_RESET_CP) {
3164 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3165
3166 srbm_soft_reset |= SOFT_RESET_GRBM;
3167 }
06bc6df0
AD		 3168
3169 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3170 srbm_soft_reset |= SOFT_RESET_DMA;
3171
3172 if (reset_mask & RADEON_RESET_DMA1)
3173 srbm_soft_reset |= SOFT_RESET_DMA1;
3174
3175 if (reset_mask & RADEON_RESET_DISPLAY)
3176 srbm_soft_reset |= SOFT_RESET_DC;
3177
3178 if (reset_mask & RADEON_RESET_RLC)
3179 grbm_soft_reset |= SOFT_RESET_RLC;
3180
3181 if (reset_mask & RADEON_RESET_SEM)
3182 srbm_soft_reset |= SOFT_RESET_SEM;
3183
3184 if (reset_mask & RADEON_RESET_IH)
3185 srbm_soft_reset |= SOFT_RESET_IH;
3186
3187 if (reset_mask & RADEON_RESET_GRBM)
3188 srbm_soft_reset |= SOFT_RESET_GRBM;
3189
3190 if (reset_mask & RADEON_RESET_VMC)
3191 srbm_soft_reset |= SOFT_RESET_VMC;
3192
3193 if (reset_mask & RADEON_RESET_MC)
3194 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3195
3196 if (grbm_soft_reset) {
3197 tmp = RREG32(GRBM_SOFT_RESET);
3198 tmp |= grbm_soft_reset;
3199 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3200 WREG32(GRBM_SOFT_RESET, tmp);
3201 tmp = RREG32(GRBM_SOFT_RESET);
3202
3203 udelay(50);
3204
3205 tmp &= ~grbm_soft_reset;
3206 WREG32(GRBM_SOFT_RESET, tmp);
3207 tmp = RREG32(GRBM_SOFT_RESET);
3208 }
3209
3210 if (srbm_soft_reset) {
3211 tmp = RREG32(SRBM_SOFT_RESET);
3212 tmp |= srbm_soft_reset;
3213 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3214 WREG32(SRBM_SOFT_RESET, tmp);
3215 tmp = RREG32(SRBM_SOFT_RESET);
3216
3217 udelay(50);
3218
3219 tmp &= ~srbm_soft_reset;
3220 WREG32(SRBM_SOFT_RESET, tmp);
3221 tmp = RREG32(SRBM_SOFT_RESET);
3222 }
06bc6df0
AD		 3223
3224 /* Wait a little for things to settle down */
3225 udelay(50);
3226
c476dde2 3227 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3228 udelay(50);
3229
1c534671 3230 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3231}
3232
3233int si_asic_reset(struct radeon_device *rdev)
3234{
014bb209
AD		 3235 u32 reset_mask;
3236
3237 reset_mask = si_gpu_check_soft_reset(rdev);
3238
3239 if (reset_mask)
3240 r600_set_bios_scratch_engine_hung(rdev, true);
3241
3242 si_gpu_soft_reset(rdev, reset_mask);
3243
3244 reset_mask = si_gpu_check_soft_reset(rdev);
3245
3246 if (!reset_mask)
3247 r600_set_bios_scratch_engine_hung(rdev, false);
3248
3249 return 0;
c476dde2
AD		 3250}
3251
123bc183
AD		 3252/**
3253 * si_gfx_is_lockup - Check if the GFX engine is locked up
3254 *
3255 * @rdev: radeon_device pointer
3256 * @ring: radeon_ring structure holding ring information
3257 *
3258 * Check if the GFX engine is locked up.
3259 * Returns true if the engine appears to be locked up, false if not.
3260 */
3261bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3262{
3263 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3264
3265 if (!(reset_mask & (RADEON_RESET_GFX |
3266 RADEON_RESET_COMPUTE |
3267 RADEON_RESET_CP))) {
3268 radeon_ring_lockup_update(ring);
3269 return false;
3270 }
3271 /* force CP activities */
3272 radeon_ring_force_activity(rdev, ring);
3273 return radeon_ring_test_lockup(rdev, ring);
3274}
3275
3276/**
3277 * si_dma_is_lockup - Check if the DMA engine is locked up
3278 *
3279 * @rdev: radeon_device pointer
3280 * @ring: radeon_ring structure holding ring information
3281 *
3282 * Check if the async DMA engine is locked up.
3283 * Returns true if the engine appears to be locked up, false if not.
3284 */
3285bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3286{
3287 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3288 u32 mask;
3289
3290 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3291 mask = RADEON_RESET_DMA;
3292 else
3293 mask = RADEON_RESET_DMA1;
3294
3295 if (!(reset_mask & mask)) {
3296 radeon_ring_lockup_update(ring);
3297 return false;
3298 }
3299 /* force ring activities */
3300 radeon_ring_force_activity(rdev, ring);
3301 return radeon_ring_test_lockup(rdev, ring);
3302}
3303
d2800ee5
AD		 3304/* MC */
3305static void si_mc_program(struct radeon_device *rdev)
3306{
3307 struct evergreen_mc_save save;
3308 u32 tmp;
3309 int i, j;
3310
3311 /* Initialize HDP */
3312 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3313 WREG32((0x2c14 + j), 0x00000000);
3314 WREG32((0x2c18 + j), 0x00000000);
3315 WREG32((0x2c1c + j), 0x00000000);
3316 WREG32((0x2c20 + j), 0x00000000);
3317 WREG32((0x2c24 + j), 0x00000000);
3318 }
3319 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3320
3321 evergreen_mc_stop(rdev, &save);
3322 if (radeon_mc_wait_for_idle(rdev)) {
3323 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3324 }
3325 /* Lockout access through VGA aperture*/
3326 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3327 /* Update configuration */
3328 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3329 rdev->mc.vram_start >> 12);
3330 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3331 rdev->mc.vram_end >> 12);
3332 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3333 rdev->vram_scratch.gpu_addr >> 12);
3334 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3335 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3336 WREG32(MC_VM_FB_LOCATION, tmp);
3337 /* XXX double check these! */
3338 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3339 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3340 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3341 WREG32(MC_VM_AGP_BASE, 0);
3342 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3343 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3344 if (radeon_mc_wait_for_idle(rdev)) {
3345 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3346 }
3347 evergreen_mc_resume(rdev, &save);
3348 /* we need to own VRAM, so turn off the VGA renderer here
3349 * to stop it overwriting our objects */
3350 rv515_vga_render_disable(rdev);
3351}
3352
d2800ee5
AD		 3353static void si_vram_gtt_location(struct radeon_device *rdev,
3354 struct radeon_mc *mc)
3355{
3356 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3357 /* leave room for at least 1024M GTT */
3358 dev_warn(rdev->dev, "limiting VRAM\n");
3359 mc->real_vram_size = 0xFFC0000000ULL;
3360 mc->mc_vram_size = 0xFFC0000000ULL;
3361 }
9ed8b1f9 3362 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3363 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3364 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3365}
3366
3367static int si_mc_init(struct radeon_device *rdev)
3368{
3369 u32 tmp;
3370 int chansize, numchan;
3371
3372 /* Get VRAM informations */
3373 rdev->mc.vram_is_ddr = true;
3374 tmp = RREG32(MC_ARB_RAMCFG);
3375 if (tmp & CHANSIZE_OVERRIDE) {
3376 chansize = 16;
3377 } else if (tmp & CHANSIZE_MASK) {
3378 chansize = 64;
3379 } else {
3380 chansize = 32;
3381 }
3382 tmp = RREG32(MC_SHARED_CHMAP);
3383 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3384 case 0:
3385 default:
3386 numchan = 1;
3387 break;
3388 case 1:
3389 numchan = 2;
3390 break;
3391 case 2:
3392 numchan = 4;
3393 break;
3394 case 3:
3395 numchan = 8;
3396 break;
3397 case 4:
3398 numchan = 3;
3399 break;
3400 case 5:
3401 numchan = 6;
3402 break;
3403 case 6:
3404 numchan = 10;
3405 break;
3406 case 7:
3407 numchan = 12;
3408 break;
3409 case 8:
3410 numchan = 16;
3411 break;
3412 }
3413 rdev->mc.vram_width = numchan * chansize;
3414 /* Could aper size report 0 ? */
3415 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3416 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3417 /* size in MB on si */
fc986034
NOS		 3418 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3419 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
d2800ee5
AD		 3420 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3421 si_vram_gtt_location(rdev, &rdev->mc);
3422 radeon_update_bandwidth_info(rdev);
3423
3424 return 0;
3425}
3426
3427/*
3428 * GART
3429 */
3430void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3431{
3432 /* flush hdp cache */
3433 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3434
3435 /* bits 0-15 are the VM contexts0-15 */
3436 WREG32(VM_INVALIDATE_REQUEST, 1);
3437}
3438
1109ca09 3439static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 3440{
3441 int r, i;
3442
3443 if (rdev->gart.robj == NULL) {
3444 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3445 return -EINVAL;
3446 }
3447 r = radeon_gart_table_vram_pin(rdev);
3448 if (r)
3449 return r;
3450 radeon_gart_restore(rdev);
3451 /* Setup TLB control */
3452 WREG32(MC_VM_MX_L1_TLB_CNTL,
3453 (0xA << 7) |
3454 ENABLE_L1_TLB |
3455 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3456 ENABLE_ADVANCED_DRIVER_MODEL |
3457 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3458 /* Setup L2 cache */
3459 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3460 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3461 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3462 EFFECTIVE_L2_QUEUE_SIZE(7) |
3463 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3464 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3465 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3466 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3467 /* setup context0 */
3468 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3469 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3470 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3471 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3472 (u32)(rdev->dummy_page.addr >> 12));
3473 WREG32(VM_CONTEXT0_CNTL2, 0);
3474 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3475 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3476
3477 WREG32(0x15D4, 0);
3478 WREG32(0x15D8, 0);
3479 WREG32(0x15DC, 0);
3480
3481 /* empty context1-15 */
d2800ee5
AD		 3482 /* set vm size, must be a multiple of 4 */
3483 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 3484 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 3485 /* Assign the pt base to something valid for now; the pts used for
3486 * the VMs are determined by the application and setup and assigned
3487 * on the fly in the vm part of radeon_gart.c
3488 */
d2800ee5
AD		 3489 for (i = 1; i < 16; i++) {
3490 if (i < 8)
3491 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3492 rdev->gart.table_addr >> 12);
3493 else
3494 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3495 rdev->gart.table_addr >> 12);
3496 }
3497
3498 /* enable context1-15 */
3499 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3500 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 3501 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 3502 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 3503 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3504 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3505 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3506 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3507 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3508 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3509 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3510 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3511 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3512 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3513 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3514 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 3515
3516 si_pcie_gart_tlb_flush(rdev);
3517 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3518 (unsigned)(rdev->mc.gtt_size >> 20),
3519 (unsigned long long)rdev->gart.table_addr);
3520 rdev->gart.ready = true;
3521 return 0;
3522}
3523
1109ca09 3524static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 3525{
3526 /* Disable all tables */
3527 WREG32(VM_CONTEXT0_CNTL, 0);
3528 WREG32(VM_CONTEXT1_CNTL, 0);
3529 /* Setup TLB control */
3530 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3531 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3532 /* Setup L2 cache */
3533 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3534 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3535 EFFECTIVE_L2_QUEUE_SIZE(7) |
3536 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3537 WREG32(VM_L2_CNTL2, 0);
3538 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3539 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3540 radeon_gart_table_vram_unpin(rdev);
3541}
3542
1109ca09 3543static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 3544{
3545 si_pcie_gart_disable(rdev);
3546 radeon_gart_table_vram_free(rdev);
3547 radeon_gart_fini(rdev);
3548}
3549
498dd8b3
AD		 3550/* vm parser */
3551static bool si_vm_reg_valid(u32 reg)
3552{
3553 /* context regs are fine */
3554 if (reg >= 0x28000)
3555 return true;
3556
3557 /* check config regs */
3558 switch (reg) {
3559 case GRBM_GFX_INDEX:
f418b88a 3560 case CP_STRMOUT_CNTL:
498dd8b3
AD		 3561 case VGT_VTX_VECT_EJECT_REG:
3562 case VGT_CACHE_INVALIDATION:
3563 case VGT_ESGS_RING_SIZE:
3564 case VGT_GSVS_RING_SIZE:
3565 case VGT_GS_VERTEX_REUSE:
3566 case VGT_PRIMITIVE_TYPE:
3567 case VGT_INDEX_TYPE:
3568 case VGT_NUM_INDICES:
3569 case VGT_NUM_INSTANCES:
3570 case VGT_TF_RING_SIZE:
3571 case VGT_HS_OFFCHIP_PARAM:
3572 case VGT_TF_MEMORY_BASE:
3573 case PA_CL_ENHANCE:
3574 case PA_SU_LINE_STIPPLE_VALUE:
3575 case PA_SC_LINE_STIPPLE_STATE:
3576 case PA_SC_ENHANCE:
3577 case SQC_CACHES:
3578 case SPI_STATIC_THREAD_MGMT_1:
3579 case SPI_STATIC_THREAD_MGMT_2:
3580 case SPI_STATIC_THREAD_MGMT_3:
3581 case SPI_PS_MAX_WAVE_ID:
3582 case SPI_CONFIG_CNTL:
3583 case SPI_CONFIG_CNTL_1:
3584 case TA_CNTL_AUX:
3585 return true;
3586 default:
3587 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
3588 return false;
3589 }
3590}
3591
3592static int si_vm_packet3_ce_check(struct radeon_device *rdev,
3593 u32 *ib, struct radeon_cs_packet *pkt)
3594{
3595 switch (pkt->opcode) {
3596 case PACKET3_NOP:
3597 case PACKET3_SET_BASE:
3598 case PACKET3_SET_CE_DE_COUNTERS:
3599 case PACKET3_LOAD_CONST_RAM:
3600 case PACKET3_WRITE_CONST_RAM:
3601 case PACKET3_WRITE_CONST_RAM_OFFSET:
3602 case PACKET3_DUMP_CONST_RAM:
3603 case PACKET3_INCREMENT_CE_COUNTER:
3604 case PACKET3_WAIT_ON_DE_COUNTER:
3605 case PACKET3_CE_WRITE:
3606 break;
3607 default:
3608 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
3609 return -EINVAL;
3610 }
3611 return 0;
3612}
3613
3614static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
3615 u32 *ib, struct radeon_cs_packet *pkt)
3616{
3617 u32 idx = pkt->idx + 1;
3618 u32 idx_value = ib[idx];
3619 u32 start_reg, end_reg, reg, i;
5aa709be 3620 u32 command, info;
498dd8b3
AD		 3621
3622 switch (pkt->opcode) {
3623 case PACKET3_NOP:
3624 case PACKET3_SET_BASE:
3625 case PACKET3_CLEAR_STATE:
3626 case PACKET3_INDEX_BUFFER_SIZE:
3627 case PACKET3_DISPATCH_DIRECT:
3628 case PACKET3_DISPATCH_INDIRECT:
3629 case PACKET3_ALLOC_GDS:
3630 case PACKET3_WRITE_GDS_RAM:
3631 case PACKET3_ATOMIC_GDS:
3632 case PACKET3_ATOMIC:
3633 case PACKET3_OCCLUSION_QUERY:
3634 case PACKET3_SET_PREDICATION:
3635 case PACKET3_COND_EXEC:
3636 case PACKET3_PRED_EXEC:
3637 case PACKET3_DRAW_INDIRECT:
3638 case PACKET3_DRAW_INDEX_INDIRECT:
3639 case PACKET3_INDEX_BASE:
3640 case PACKET3_DRAW_INDEX_2:
3641 case PACKET3_CONTEXT_CONTROL:
3642 case PACKET3_INDEX_TYPE:
3643 case PACKET3_DRAW_INDIRECT_MULTI:
3644 case PACKET3_DRAW_INDEX_AUTO:
3645 case PACKET3_DRAW_INDEX_IMMD:
3646 case PACKET3_NUM_INSTANCES:
3647 case PACKET3_DRAW_INDEX_MULTI_AUTO:
3648 case PACKET3_STRMOUT_BUFFER_UPDATE:
3649 case PACKET3_DRAW_INDEX_OFFSET_2:
3650 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
3651 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
3652 case PACKET3_MPEG_INDEX:
3653 case PACKET3_WAIT_REG_MEM:
3654 case PACKET3_MEM_WRITE:
3655 case PACKET3_PFP_SYNC_ME:
3656 case PACKET3_SURFACE_SYNC:
3657 case PACKET3_EVENT_WRITE:
3658 case PACKET3_EVENT_WRITE_EOP:
3659 case PACKET3_EVENT_WRITE_EOS:
3660 case PACKET3_SET_CONTEXT_REG:
3661 case PACKET3_SET_CONTEXT_REG_INDIRECT:
3662 case PACKET3_SET_SH_REG:
3663 case PACKET3_SET_SH_REG_OFFSET:
3664 case PACKET3_INCREMENT_DE_COUNTER:
3665 case PACKET3_WAIT_ON_CE_COUNTER:
3666 case PACKET3_WAIT_ON_AVAIL_BUFFER:
3667 case PACKET3_ME_WRITE:
3668 break;
3669 case PACKET3_COPY_DATA:
3670 if ((idx_value & 0xf00) == 0) {
3671 reg = ib[idx + 3] * 4;
3672 if (!si_vm_reg_valid(reg))
3673 return -EINVAL;
3674 }
3675 break;
3676 case PACKET3_WRITE_DATA:
3677 if ((idx_value & 0xf00) == 0) {
3678 start_reg = ib[idx + 1] * 4;
3679 if (idx_value & 0x10000) {
3680 if (!si_vm_reg_valid(start_reg))
3681 return -EINVAL;
3682 } else {
3683 for (i = 0; i < (pkt->count - 2); i++) {
3684 reg = start_reg + (4 * i);
3685 if (!si_vm_reg_valid(reg))
3686 return -EINVAL;
3687 }
3688 }
3689 }
3690 break;
3691 case PACKET3_COND_WRITE:
3692 if (idx_value & 0x100) {
3693 reg = ib[idx + 5] * 4;
3694 if (!si_vm_reg_valid(reg))
3695 return -EINVAL;
3696 }
3697 break;
3698 case PACKET3_COPY_DW:
3699 if (idx_value & 0x2) {
3700 reg = ib[idx + 3] * 4;
3701 if (!si_vm_reg_valid(reg))
3702 return -EINVAL;
3703 }
3704 break;
3705 case PACKET3_SET_CONFIG_REG:
3706 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
3707 end_reg = 4 * pkt->count + start_reg - 4;
3708 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
3709 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
3710 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
3711 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
3712 return -EINVAL;
3713 }
3714 for (i = 0; i < pkt->count; i++) {
3715 reg = start_reg + (4 * i);
3716 if (!si_vm_reg_valid(reg))
3717 return -EINVAL;
3718 }
3719 break;
5aa709be
AD		 3720 case PACKET3_CP_DMA:
3721 command = ib[idx + 4];
3722 info = ib[idx + 1];
3723 if (command & PACKET3_CP_DMA_CMD_SAS) {
3724 /* src address space is register */
3725 if (((info & 0x60000000) >> 29) == 0) {
3726 start_reg = idx_value << 2;
3727 if (command & PACKET3_CP_DMA_CMD_SAIC) {
3728 reg = start_reg;
3729 if (!si_vm_reg_valid(reg)) {
3730 DRM_ERROR("CP DMA Bad SRC register\n");
3731 return -EINVAL;
3732 }
3733 } else {
3734 for (i = 0; i < (command & 0x1fffff); i++) {
3735 reg = start_reg + (4 * i);
3736 if (!si_vm_reg_valid(reg)) {
3737 DRM_ERROR("CP DMA Bad SRC register\n");
3738 return -EINVAL;
3739 }
3740 }
3741 }
3742 }
3743 }
3744 if (command & PACKET3_CP_DMA_CMD_DAS) {
3745 /* dst address space is register */
3746 if (((info & 0x00300000) >> 20) == 0) {
3747 start_reg = ib[idx + 2];
3748 if (command & PACKET3_CP_DMA_CMD_DAIC) {
3749 reg = start_reg;
3750 if (!si_vm_reg_valid(reg)) {
3751 DRM_ERROR("CP DMA Bad DST register\n");
3752 return -EINVAL;
3753 }
3754 } else {
3755 for (i = 0; i < (command & 0x1fffff); i++) {
3756 reg = start_reg + (4 * i);
3757 if (!si_vm_reg_valid(reg)) {
3758 DRM_ERROR("CP DMA Bad DST register\n");
3759 return -EINVAL;
3760 }
3761 }
3762 }
3763 }
3764 }
3765 break;
498dd8b3
AD		 3766 default:
3767 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
3768 return -EINVAL;
3769 }
3770 return 0;
3771}
3772
3773static int si_vm_packet3_compute_check(struct radeon_device *rdev,
3774 u32 *ib, struct radeon_cs_packet *pkt)
3775{
3776 u32 idx = pkt->idx + 1;
3777 u32 idx_value = ib[idx];
3778 u32 start_reg, reg, i;
3779
3780 switch (pkt->opcode) {
3781 case PACKET3_NOP:
3782 case PACKET3_SET_BASE:
3783 case PACKET3_CLEAR_STATE:
3784 case PACKET3_DISPATCH_DIRECT:
3785 case PACKET3_DISPATCH_INDIRECT:
3786 case PACKET3_ALLOC_GDS:
3787 case PACKET3_WRITE_GDS_RAM:
3788 case PACKET3_ATOMIC_GDS:
3789 case PACKET3_ATOMIC:
3790 case PACKET3_OCCLUSION_QUERY:
3791 case PACKET3_SET_PREDICATION:
3792 case PACKET3_COND_EXEC:
3793 case PACKET3_PRED_EXEC:
3794 case PACKET3_CONTEXT_CONTROL:
3795 case PACKET3_STRMOUT_BUFFER_UPDATE:
3796 case PACKET3_WAIT_REG_MEM:
3797 case PACKET3_MEM_WRITE:
3798 case PACKET3_PFP_SYNC_ME:
3799 case PACKET3_SURFACE_SYNC:
3800 case PACKET3_EVENT_WRITE:
3801 case PACKET3_EVENT_WRITE_EOP:
3802 case PACKET3_EVENT_WRITE_EOS:
3803 case PACKET3_SET_CONTEXT_REG:
3804 case PACKET3_SET_CONTEXT_REG_INDIRECT:
3805 case PACKET3_SET_SH_REG:
3806 case PACKET3_SET_SH_REG_OFFSET:
3807 case PACKET3_INCREMENT_DE_COUNTER:
3808 case PACKET3_WAIT_ON_CE_COUNTER:
3809 case PACKET3_WAIT_ON_AVAIL_BUFFER:
3810 case PACKET3_ME_WRITE:
3811 break;
3812 case PACKET3_COPY_DATA:
3813 if ((idx_value & 0xf00) == 0) {
3814 reg = ib[idx + 3] * 4;
3815 if (!si_vm_reg_valid(reg))
3816 return -EINVAL;
3817 }
3818 break;
3819 case PACKET3_WRITE_DATA:
3820 if ((idx_value & 0xf00) == 0) {
3821 start_reg = ib[idx + 1] * 4;
3822 if (idx_value & 0x10000) {
3823 if (!si_vm_reg_valid(start_reg))
3824 return -EINVAL;
3825 } else {
3826 for (i = 0; i < (pkt->count - 2); i++) {
3827 reg = start_reg + (4 * i);
3828 if (!si_vm_reg_valid(reg))
3829 return -EINVAL;
3830 }
3831 }
3832 }
3833 break;
3834 case PACKET3_COND_WRITE:
3835 if (idx_value & 0x100) {
3836 reg = ib[idx + 5] * 4;
3837 if (!si_vm_reg_valid(reg))
3838 return -EINVAL;
3839 }
3840 break;
3841 case PACKET3_COPY_DW:
3842 if (idx_value & 0x2) {
3843 reg = ib[idx + 3] * 4;
3844 if (!si_vm_reg_valid(reg))
3845 return -EINVAL;
3846 }
3847 break;
3848 default:
3849 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
3850 return -EINVAL;
3851 }
3852 return 0;
3853}
3854
3855int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
3856{
3857 int ret = 0;
3858 u32 idx = 0;
3859 struct radeon_cs_packet pkt;
3860
3861 do {
3862 pkt.idx = idx;
4e872ae2
IH		 3863 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
3864 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 3865 pkt.one_reg_wr = 0;
3866 switch (pkt.type) {
4e872ae2 3867 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 3868 dev_err(rdev->dev, "Packet0 not allowed!\n");
3869 ret = -EINVAL;
3870 break;
4e872ae2 3871 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 3872 idx += 1;
3873 break;
4e872ae2
IH		 3874 case RADEON_PACKET_TYPE3:
3875 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 3876 if (ib->is_const_ib)
3877 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
3878 else {
876dc9f3 3879 switch (ib->ring) {
498dd8b3
AD		 3880 case RADEON_RING_TYPE_GFX_INDEX:
3881 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
3882 break;
3883 case CAYMAN_RING_TYPE_CP1_INDEX:
3884 case CAYMAN_RING_TYPE_CP2_INDEX:
3885 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
3886 break;
3887 default:
876dc9f3 3888 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 3889 ret = -EINVAL;
3890 break;
3891 }
3892 }
3893 idx += pkt.count + 2;
3894 break;
3895 default:
3896 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
3897 ret = -EINVAL;
3898 break;
3899 }
3900 if (ret)
3901 break;
3902 } while (idx < ib->length_dw);
3903
3904 return ret;
3905}
3906
d2800ee5
AD		 3907/*
3908 * vm
3909 */
3910int si_vm_init(struct radeon_device *rdev)
3911{
3912 /* number of VMs */
3913 rdev->vm_manager.nvm = 16;
3914 /* base offset of vram pages */
3915 rdev->vm_manager.vram_base_offset = 0;
3916
3917 return 0;
3918}
3919
3920void si_vm_fini(struct radeon_device *rdev)
3921{
3922}
3923
82ffd92b
AD		 3924/**
3925 * si_vm_set_page - update the page tables using the CP
3926 *
3927 * @rdev: radeon_device pointer
43f1214a 3928 * @ib: indirect buffer to fill with commands
82ffd92b
AD		 3929 * @pe: addr of the page entry
3930 * @addr: dst addr to write into pe
3931 * @count: number of page entries to update
3932 * @incr: increase next addr by incr bytes
3933 * @flags: access flags
3934 *
43f1214a 3935 * Update the page tables using the CP (SI).
82ffd92b 3936 */
43f1214a
AD		 3937void si_vm_set_page(struct radeon_device *rdev,
3938 struct radeon_ib *ib,
3939 uint64_t pe,
82ffd92b
AD		 3940 uint64_t addr, unsigned count,
3941 uint32_t incr, uint32_t flags)
d2800ee5 3942{
82ffd92b 3943 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
deab48f1
AD		 3944 uint64_t value;
3945 unsigned ndw;
3946
3947 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
3948 while (count) {
3949 ndw = 2 + count * 2;
3950 if (ndw > 0x3FFE)
3951 ndw = 0x3FFE;
3952
43f1214a
AD		 3953 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
3954 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
3955 WRITE_DATA_DST_SEL(1));
3956 ib->ptr[ib->length_dw++] = pe;
3957 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
deab48f1
AD		 3958 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
3959 if (flags & RADEON_VM_PAGE_SYSTEM) {
3960 value = radeon_vm_map_gart(rdev, addr);
3961 value &= 0xFFFFFFFFFFFFF000ULL;
3962 } else if (flags & RADEON_VM_PAGE_VALID) {
3963 value = addr;
3964 } else {
3965 value = 0;
3966 }
3967 addr += incr;
3968 value |= r600_flags;
43f1214a
AD		 3969 ib->ptr[ib->length_dw++] = value;
3970 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 3971 }
3972 }
3973 } else {
3974 /* DMA */
3975 if (flags & RADEON_VM_PAGE_SYSTEM) {
3976 while (count) {
3977 ndw = count * 2;
3978 if (ndw > 0xFFFFE)
3979 ndw = 0xFFFFE;
3980
3981 /* for non-physically contiguous pages (system) */
43f1214a
AD		 3982 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
3983 ib->ptr[ib->length_dw++] = pe;
3984 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
deab48f1
AD		 3985 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
3986 if (flags & RADEON_VM_PAGE_SYSTEM) {
3987 value = radeon_vm_map_gart(rdev, addr);
3988 value &= 0xFFFFFFFFFFFFF000ULL;
3989 } else if (flags & RADEON_VM_PAGE_VALID) {
3990 value = addr;
3991 } else {
3992 value = 0;
3993 }
3994 addr += incr;
3995 value |= r600_flags;
43f1214a
AD		 3996 ib->ptr[ib->length_dw++] = value;
3997 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 3998 }
3999 }
4000 } else {
4001 while (count) {
4002 ndw = count * 2;
4003 if (ndw > 0xFFFFE)
4004 ndw = 0xFFFFE;
4005
4006 if (flags & RADEON_VM_PAGE_VALID)
4007 value = addr;
4008 else
4009 value = 0;
4010 /* for physically contiguous pages (vram) */
43f1214a
AD		 4011 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
4012 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4013 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4014 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4015 ib->ptr[ib->length_dw++] = 0;
4016 ib->ptr[ib->length_dw++] = value; /* value */
4017 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4018 ib->ptr[ib->length_dw++] = incr; /* increment size */
4019 ib->ptr[ib->length_dw++] = 0;
deab48f1
AD		 4020 pe += ndw * 4;
4021 addr += (ndw / 2) * incr;
4022 count -= ndw / 2;
4023 }
d7025d89 4024 }
43f1214a
AD		 4025 while (ib->length_dw & 0x7)
4026 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
82ffd92b 4027 }
d2800ee5
AD		 4028}
4029
498522b4 4030void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4031{
498522b4 4032 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4033
ee60e29f 4034 if (vm == NULL)
d2800ee5
AD		 4035 return;
4036
76c44f2c
AD		 4037 /* write new base address */
4038 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4039 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4040 WRITE_DATA_DST_SEL(0)));
4041
ee60e29f 4042 if (vm->id < 8) {
76c44f2c
AD		 4043 radeon_ring_write(ring,
4044 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4045 } else {
76c44f2c
AD		 4046 radeon_ring_write(ring,
4047 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4048 }
76c44f2c 4049 radeon_ring_write(ring, 0);
fa87e62d 4050 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4051
d2800ee5 4052 /* flush hdp cache */
76c44f2c
AD		 4053 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4054 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4055 WRITE_DATA_DST_SEL(0)));
4056 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4057 radeon_ring_write(ring, 0);
ee60e29f
CK		 4058 radeon_ring_write(ring, 0x1);
4059
d2800ee5 4060 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4061 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4062 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4063 WRITE_DATA_DST_SEL(0)));
4064 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4065 radeon_ring_write(ring, 0);
498522b4 4066 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4067
4068 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4069 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4070 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4071}
4072
8c5fd7ef
AD		 4073void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4074{
4075 struct radeon_ring *ring = &rdev->ring[ridx];
4076
4077 if (vm == NULL)
4078 return;
4079
4080 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4081 if (vm->id < 8) {
4082 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4083 } else {
4084 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4085 }
4086 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4087
4088 /* flush hdp cache */
4089 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4090 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4091 radeon_ring_write(ring, 1);
4092
4093 /* bits 0-7 are the VM contexts0-7 */
4094 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4095 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4096 radeon_ring_write(ring, 1 << vm->id);
4097}
4098
347e7592
AD		 4099/*
4100 * RLC
4101 */
c420c745 4102void si_rlc_fini(struct radeon_device *rdev)
347e7592
AD		 4103{
4104 int r;
4105
4106 /* save restore block */
4107 if (rdev->rlc.save_restore_obj) {
4108 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4109 if (unlikely(r != 0))
4110 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4111 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4112 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4113
4114 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4115 rdev->rlc.save_restore_obj = NULL;
4116 }
4117
4118 /* clear state block */
4119 if (rdev->rlc.clear_state_obj) {
4120 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4121 if (unlikely(r != 0))
4122 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4123 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4124 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4125
4126 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4127 rdev->rlc.clear_state_obj = NULL;
4128 }
4129}
4130
c420c745 4131int si_rlc_init(struct radeon_device *rdev)
347e7592
AD		 4132{
4133 int r;
4134
4135 /* save restore block */
4136 if (rdev->rlc.save_restore_obj == NULL) {
4137 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
40f5cf99
AD		 4138 RADEON_GEM_DOMAIN_VRAM, NULL,
4139 &rdev->rlc.save_restore_obj);
347e7592
AD		 4140 if (r) {
4141 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4142 return r;
4143 }
4144 }
4145
4146 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4147 if (unlikely(r != 0)) {
4148 si_rlc_fini(rdev);
4149 return r;
4150 }
4151 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4152 &rdev->rlc.save_restore_gpu_addr);
5273db70 4153 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
347e7592 4154 if (r) {
347e7592
AD		 4155 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4156 si_rlc_fini(rdev);
4157 return r;
4158 }
4159
4160 /* clear state block */
4161 if (rdev->rlc.clear_state_obj == NULL) {
4162 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
40f5cf99
AD		 4163 RADEON_GEM_DOMAIN_VRAM, NULL,
4164 &rdev->rlc.clear_state_obj);
347e7592
AD		 4165 if (r) {
4166 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4167 si_rlc_fini(rdev);
4168 return r;
4169 }
4170 }
4171 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4172 if (unlikely(r != 0)) {
4173 si_rlc_fini(rdev);
4174 return r;
4175 }
4176 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4177 &rdev->rlc.clear_state_gpu_addr);
5273db70 4178 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
347e7592 4179 if (r) {
347e7592
AD		 4180 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4181 si_rlc_fini(rdev);
4182 return r;
4183 }
4184
4185 return 0;
4186}
4187
4188static void si_rlc_stop(struct radeon_device *rdev)
4189{
4190 WREG32(RLC_CNTL, 0);
4191}
4192
4193static void si_rlc_start(struct radeon_device *rdev)
4194{
4195 WREG32(RLC_CNTL, RLC_ENABLE);
4196}
4197
4198static int si_rlc_resume(struct radeon_device *rdev)
4199{
4200 u32 i;
4201 const __be32 *fw_data;
4202
4203 if (!rdev->rlc_fw)
4204 return -EINVAL;
4205
4206 si_rlc_stop(rdev);
4207
4208 WREG32(RLC_RL_BASE, 0);
4209 WREG32(RLC_RL_SIZE, 0);
4210 WREG32(RLC_LB_CNTL, 0);
4211 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
4212 WREG32(RLC_LB_CNTR_INIT, 0);
4213
4214 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4215 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4216
4217 WREG32(RLC_MC_CNTL, 0);
4218 WREG32(RLC_UCODE_CNTL, 0);
4219
4220 fw_data = (const __be32 *)rdev->rlc_fw->data;
4221 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
4222 WREG32(RLC_UCODE_ADDR, i);
4223 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4224 }
4225 WREG32(RLC_UCODE_ADDR, 0);
4226
4227 si_rlc_start(rdev);
4228
4229 return 0;
4230}
4231
25a857fb
AD		 4232static void si_enable_interrupts(struct radeon_device *rdev)
4233{
4234 u32 ih_cntl = RREG32(IH_CNTL);
4235 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4236
4237 ih_cntl |= ENABLE_INTR;
4238 ih_rb_cntl |= IH_RB_ENABLE;
4239 WREG32(IH_CNTL, ih_cntl);
4240 WREG32(IH_RB_CNTL, ih_rb_cntl);
4241 rdev->ih.enabled = true;
4242}
4243
4244static void si_disable_interrupts(struct radeon_device *rdev)
4245{
4246 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4247 u32 ih_cntl = RREG32(IH_CNTL);
4248
4249 ih_rb_cntl &= ~IH_RB_ENABLE;
4250 ih_cntl &= ~ENABLE_INTR;
4251 WREG32(IH_RB_CNTL, ih_rb_cntl);
4252 WREG32(IH_CNTL, ih_cntl);
4253 /* set rptr, wptr to 0 */
4254 WREG32(IH_RB_RPTR, 0);
4255 WREG32(IH_RB_WPTR, 0);
4256 rdev->ih.enabled = false;
25a857fb
AD		 4257 rdev->ih.rptr = 0;
4258}
4259
4260static void si_disable_interrupt_state(struct radeon_device *rdev)
4261{
4262 u32 tmp;
4263
4264 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4265 WREG32(CP_INT_CNTL_RING1, 0);
4266 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 4267 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4268 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
4269 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4270 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb
AD		 4271 WREG32(GRBM_INT_CNTL, 0);
4272 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4273 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4274 if (rdev->num_crtc >= 4) {
4275 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4276 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4277 }
4278 if (rdev->num_crtc >= 6) {
4279 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4280 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4281 }
4282
4283 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4284 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4285 if (rdev->num_crtc >= 4) {
4286 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4287 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4288 }
4289 if (rdev->num_crtc >= 6) {
4290 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4291 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4292 }
4293
4294 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4295
4296 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4297 WREG32(DC_HPD1_INT_CONTROL, tmp);
4298 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4299 WREG32(DC_HPD2_INT_CONTROL, tmp);
4300 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4301 WREG32(DC_HPD3_INT_CONTROL, tmp);
4302 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4303 WREG32(DC_HPD4_INT_CONTROL, tmp);
4304 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4305 WREG32(DC_HPD5_INT_CONTROL, tmp);
4306 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4307 WREG32(DC_HPD6_INT_CONTROL, tmp);
4308
4309}
4310
4311static int si_irq_init(struct radeon_device *rdev)
4312{
4313 int ret = 0;
4314 int rb_bufsz;
4315 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
4316
4317 /* allocate ring */
4318 ret = r600_ih_ring_alloc(rdev);
4319 if (ret)
4320 return ret;
4321
4322 /* disable irqs */
4323 si_disable_interrupts(rdev);
4324
4325 /* init rlc */
4326 ret = si_rlc_resume(rdev);
4327 if (ret) {
4328 r600_ih_ring_fini(rdev);
4329 return ret;
4330 }
4331
4332 /* setup interrupt control */
4333 /* set dummy read address to ring address */
4334 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
4335 interrupt_cntl = RREG32(INTERRUPT_CNTL);
4336 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
4337 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
4338 */
4339 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
4340 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
4341 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
4342 WREG32(INTERRUPT_CNTL, interrupt_cntl);
4343
4344 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
4345 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
4346
4347 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
4348 IH_WPTR_OVERFLOW_CLEAR |
4349 (rb_bufsz << 1));
4350
4351 if (rdev->wb.enabled)
4352 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
4353
4354 /* set the writeback address whether it's enabled or not */
4355 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
4356 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
4357
4358 WREG32(IH_RB_CNTL, ih_rb_cntl);
4359
4360 /* set rptr, wptr to 0 */
4361 WREG32(IH_RB_RPTR, 0);
4362 WREG32(IH_RB_WPTR, 0);
4363
4364 /* Default settings for IH_CNTL (disabled at first) */
4365 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
4366 /* RPTR_REARM only works if msi's are enabled */
4367 if (rdev->msi_enabled)
4368 ih_cntl |= RPTR_REARM;
4369 WREG32(IH_CNTL, ih_cntl);
4370
4371 /* force the active interrupt state to all disabled */
4372 si_disable_interrupt_state(rdev);
4373
2099810f
DA		 4374 pci_set_master(rdev->pdev);
4375
25a857fb
AD		 4376 /* enable irqs */
4377 si_enable_interrupts(rdev);
4378
4379 return ret;
4380}
4381
4382int si_irq_set(struct radeon_device *rdev)
4383{
4384 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4385 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4386 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4387 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4388 u32 grbm_int_cntl = 0;
4389 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 4390 u32 dma_cntl, dma_cntl1;
25a857fb
AD		 4391
4392 if (!rdev->irq.installed) {
4393 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4394 return -EINVAL;
4395 }
4396 /* don't enable anything if the ih is disabled */
4397 if (!rdev->ih.enabled) {
4398 si_disable_interrupts(rdev);
4399 /* force the active interrupt state to all disabled */
4400 si_disable_interrupt_state(rdev);
4401 return 0;
4402 }
4403
4404 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4405 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4406 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4407 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4408 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4409 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4410
8c5fd7ef
AD		 4411 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4412 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4413
25a857fb 4414 /* enable CP interrupts on all rings */
736fc37f 4415 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 4416 DRM_DEBUG("si_irq_set: sw int gfx\n");
4417 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4418 }
736fc37f 4419 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 4420 DRM_DEBUG("si_irq_set: sw int cp1\n");
4421 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4422 }
736fc37f 4423 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 4424 DRM_DEBUG("si_irq_set: sw int cp2\n");
4425 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4426 }
8c5fd7ef
AD		 4427 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4428 DRM_DEBUG("si_irq_set: sw int dma\n");
4429 dma_cntl |= TRAP_ENABLE;
4430 }
4431
4432 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4433 DRM_DEBUG("si_irq_set: sw int dma1\n");
4434 dma_cntl1 |= TRAP_ENABLE;
4435 }
25a857fb 4436 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4437 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 4438 DRM_DEBUG("si_irq_set: vblank 0\n");
4439 crtc1 |= VBLANK_INT_MASK;
4440 }
4441 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4442 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 4443 DRM_DEBUG("si_irq_set: vblank 1\n");
4444 crtc2 |= VBLANK_INT_MASK;
4445 }
4446 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4447 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 4448 DRM_DEBUG("si_irq_set: vblank 2\n");
4449 crtc3 |= VBLANK_INT_MASK;
4450 }
4451 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4452 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 4453 DRM_DEBUG("si_irq_set: vblank 3\n");
4454 crtc4 |= VBLANK_INT_MASK;
4455 }
4456 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4457 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 4458 DRM_DEBUG("si_irq_set: vblank 4\n");
4459 crtc5 |= VBLANK_INT_MASK;
4460 }
4461 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4462 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 4463 DRM_DEBUG("si_irq_set: vblank 5\n");
4464 crtc6 |= VBLANK_INT_MASK;
4465 }
4466 if (rdev->irq.hpd[0]) {
4467 DRM_DEBUG("si_irq_set: hpd 1\n");
4468 hpd1 |= DC_HPDx_INT_EN;
4469 }
4470 if (rdev->irq.hpd[1]) {
4471 DRM_DEBUG("si_irq_set: hpd 2\n");
4472 hpd2 |= DC_HPDx_INT_EN;
4473 }
4474 if (rdev->irq.hpd[2]) {
4475 DRM_DEBUG("si_irq_set: hpd 3\n");
4476 hpd3 |= DC_HPDx_INT_EN;
4477 }
4478 if (rdev->irq.hpd[3]) {
4479 DRM_DEBUG("si_irq_set: hpd 4\n");
4480 hpd4 |= DC_HPDx_INT_EN;
4481 }
4482 if (rdev->irq.hpd[4]) {
4483 DRM_DEBUG("si_irq_set: hpd 5\n");
4484 hpd5 |= DC_HPDx_INT_EN;
4485 }
4486 if (rdev->irq.hpd[5]) {
4487 DRM_DEBUG("si_irq_set: hpd 6\n");
4488 hpd6 |= DC_HPDx_INT_EN;
4489 }
25a857fb
AD		 4490
4491 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
4492 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
4493 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
4494
8c5fd7ef
AD		 4495 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
4496 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
4497
25a857fb
AD		 4498 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4499
4500 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4501 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4502 if (rdev->num_crtc >= 4) {
4503 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4504 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4505 }
4506 if (rdev->num_crtc >= 6) {
4507 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4508 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4509 }
4510
4511 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4512 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4513 if (rdev->num_crtc >= 4) {
4514 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4515 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4516 }
4517 if (rdev->num_crtc >= 6) {
4518 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4519 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4520 }
4521
4522 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4523 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4524 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4525 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4526 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4527 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4528
4529 return 0;
4530}
4531
4532static inline void si_irq_ack(struct radeon_device *rdev)
4533{
4534 u32 tmp;
4535
4536 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4537 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4538 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4539 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4540 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4541 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4542 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4543 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4544 if (rdev->num_crtc >= 4) {
4545 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4546 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4547 }
4548 if (rdev->num_crtc >= 6) {
4549 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4550 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4551 }
4552
4553 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4554 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4555 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4556 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4557 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4558 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4559 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4560 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4561 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4562 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4563 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4564 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4565
4566 if (rdev->num_crtc >= 4) {
4567 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4568 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4569 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4570 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4571 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4572 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4573 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4574 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4575 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4576 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4577 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4578 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4579 }
4580
4581 if (rdev->num_crtc >= 6) {
4582 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4583 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4584 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4585 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4586 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4587 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4588 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4589 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4590 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4591 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4592 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4593 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4594 }
4595
4596 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4597 tmp = RREG32(DC_HPD1_INT_CONTROL);
4598 tmp |= DC_HPDx_INT_ACK;
4599 WREG32(DC_HPD1_INT_CONTROL, tmp);
4600 }
4601 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4602 tmp = RREG32(DC_HPD2_INT_CONTROL);
4603 tmp |= DC_HPDx_INT_ACK;
4604 WREG32(DC_HPD2_INT_CONTROL, tmp);
4605 }
4606 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4607 tmp = RREG32(DC_HPD3_INT_CONTROL);
4608 tmp |= DC_HPDx_INT_ACK;
4609 WREG32(DC_HPD3_INT_CONTROL, tmp);
4610 }
4611 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4612 tmp = RREG32(DC_HPD4_INT_CONTROL);
4613 tmp |= DC_HPDx_INT_ACK;
4614 WREG32(DC_HPD4_INT_CONTROL, tmp);
4615 }
4616 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4617 tmp = RREG32(DC_HPD5_INT_CONTROL);
4618 tmp |= DC_HPDx_INT_ACK;
4619 WREG32(DC_HPD5_INT_CONTROL, tmp);
4620 }
4621 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4622 tmp = RREG32(DC_HPD5_INT_CONTROL);
4623 tmp |= DC_HPDx_INT_ACK;
4624 WREG32(DC_HPD6_INT_CONTROL, tmp);
4625 }
4626}
4627
4628static void si_irq_disable(struct radeon_device *rdev)
4629{
4630 si_disable_interrupts(rdev);
4631 /* Wait and acknowledge irq */
4632 mdelay(1);
4633 si_irq_ack(rdev);
4634 si_disable_interrupt_state(rdev);
4635}
4636
4637static void si_irq_suspend(struct radeon_device *rdev)
4638{
4639 si_irq_disable(rdev);
4640 si_rlc_stop(rdev);
4641}
4642
9b136d51
AD		 4643static void si_irq_fini(struct radeon_device *rdev)
4644{
4645 si_irq_suspend(rdev);
4646 r600_ih_ring_fini(rdev);
4647}
4648
25a857fb
AD		 4649static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
4650{
4651 u32 wptr, tmp;
4652
4653 if (rdev->wb.enabled)
4654 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4655 else
4656 wptr = RREG32(IH_RB_WPTR);
4657
4658 if (wptr & RB_OVERFLOW) {
4659 /* When a ring buffer overflow happen start parsing interrupt
4660 * from the last not overwritten vector (wptr + 16). Hopefully
4661 * this should allow us to catchup.
4662 */
4663 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4664 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4665 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4666 tmp = RREG32(IH_RB_CNTL);
4667 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4668 WREG32(IH_RB_CNTL, tmp);
4669 }
4670 return (wptr & rdev->ih.ptr_mask);
4671}
4672
4673/* SI IV Ring
4674 * Each IV ring entry is 128 bits:
4675 * [7:0] - interrupt source id
4676 * [31:8] - reserved
4677 * [59:32] - interrupt source data
4678 * [63:60] - reserved
4679 * [71:64] - RINGID
4680 * [79:72] - VMID
4681 * [127:80] - reserved
4682 */
4683int si_irq_process(struct radeon_device *rdev)
4684{
4685 u32 wptr;
4686 u32 rptr;
4687 u32 src_id, src_data, ring_id;
4688 u32 ring_index;
25a857fb
AD		 4689 bool queue_hotplug = false;
4690
4691 if (!rdev->ih.enabled || rdev->shutdown)
4692 return IRQ_NONE;
4693
4694 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 4695
4696restart_ih:
4697 /* is somebody else already processing irqs? */
4698 if (atomic_xchg(&rdev->ih.lock, 1))
4699 return IRQ_NONE;
4700
25a857fb
AD		 4701 rptr = rdev->ih.rptr;
4702 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4703
25a857fb
AD		 4704 /* Order reading of wptr vs. reading of IH ring data */
4705 rmb();
4706
4707 /* display interrupts */
4708 si_irq_ack(rdev);
4709
25a857fb
AD		 4710 while (rptr != wptr) {
4711 /* wptr/rptr are in bytes! */
4712 ring_index = rptr / 4;
4713 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4714 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4715 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
4716
4717 switch (src_id) {
4718 case 1: /* D1 vblank/vline */
4719 switch (src_data) {
4720 case 0: /* D1 vblank */
4721 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4722 if (rdev->irq.crtc_vblank_int[0]) {
4723 drm_handle_vblank(rdev->ddev, 0);
4724 rdev->pm.vblank_sync = true;
4725 wake_up(&rdev->irq.vblank_queue);
4726 }
736fc37f 4727 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 4728 radeon_crtc_handle_flip(rdev, 0);
4729 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4730 DRM_DEBUG("IH: D1 vblank\n");
4731 }
4732 break;
4733 case 1: /* D1 vline */
4734 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4735 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4736 DRM_DEBUG("IH: D1 vline\n");
4737 }
4738 break;
4739 default:
4740 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4741 break;
4742 }
4743 break;
4744 case 2: /* D2 vblank/vline */
4745 switch (src_data) {
4746 case 0: /* D2 vblank */
4747 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4748 if (rdev->irq.crtc_vblank_int[1]) {
4749 drm_handle_vblank(rdev->ddev, 1);
4750 rdev->pm.vblank_sync = true;
4751 wake_up(&rdev->irq.vblank_queue);
4752 }
736fc37f 4753 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 4754 radeon_crtc_handle_flip(rdev, 1);
4755 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4756 DRM_DEBUG("IH: D2 vblank\n");
4757 }
4758 break;
4759 case 1: /* D2 vline */
4760 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4761 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4762 DRM_DEBUG("IH: D2 vline\n");
4763 }
4764 break;
4765 default:
4766 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4767 break;
4768 }
4769 break;
4770 case 3: /* D3 vblank/vline */
4771 switch (src_data) {
4772 case 0: /* D3 vblank */
4773 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4774 if (rdev->irq.crtc_vblank_int[2]) {
4775 drm_handle_vblank(rdev->ddev, 2);
4776 rdev->pm.vblank_sync = true;
4777 wake_up(&rdev->irq.vblank_queue);
4778 }
736fc37f 4779 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 4780 radeon_crtc_handle_flip(rdev, 2);
4781 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4782 DRM_DEBUG("IH: D3 vblank\n");
4783 }
4784 break;
4785 case 1: /* D3 vline */
4786 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4787 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4788 DRM_DEBUG("IH: D3 vline\n");
4789 }
4790 break;
4791 default:
4792 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4793 break;
4794 }
4795 break;
4796 case 4: /* D4 vblank/vline */
4797 switch (src_data) {
4798 case 0: /* D4 vblank */
4799 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4800 if (rdev->irq.crtc_vblank_int[3]) {
4801 drm_handle_vblank(rdev->ddev, 3);
4802 rdev->pm.vblank_sync = true;
4803 wake_up(&rdev->irq.vblank_queue);
4804 }
736fc37f 4805 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 4806 radeon_crtc_handle_flip(rdev, 3);
4807 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4808 DRM_DEBUG("IH: D4 vblank\n");
4809 }
4810 break;
4811 case 1: /* D4 vline */
4812 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4813 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4814 DRM_DEBUG("IH: D4 vline\n");
4815 }
4816 break;
4817 default:
4818 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4819 break;
4820 }
4821 break;
4822 case 5: /* D5 vblank/vline */
4823 switch (src_data) {
4824 case 0: /* D5 vblank */
4825 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4826 if (rdev->irq.crtc_vblank_int[4]) {
4827 drm_handle_vblank(rdev->ddev, 4);
4828 rdev->pm.vblank_sync = true;
4829 wake_up(&rdev->irq.vblank_queue);
4830 }
736fc37f 4831 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 4832 radeon_crtc_handle_flip(rdev, 4);
4833 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4834 DRM_DEBUG("IH: D5 vblank\n");
4835 }
4836 break;
4837 case 1: /* D5 vline */
4838 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4839 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4840 DRM_DEBUG("IH: D5 vline\n");
4841 }
4842 break;
4843 default:
4844 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4845 break;
4846 }
4847 break;
4848 case 6: /* D6 vblank/vline */
4849 switch (src_data) {
4850 case 0: /* D6 vblank */
4851 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4852 if (rdev->irq.crtc_vblank_int[5]) {
4853 drm_handle_vblank(rdev->ddev, 5);
4854 rdev->pm.vblank_sync = true;
4855 wake_up(&rdev->irq.vblank_queue);
4856 }
736fc37f 4857 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 4858 radeon_crtc_handle_flip(rdev, 5);
4859 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4860 DRM_DEBUG("IH: D6 vblank\n");
4861 }
4862 break;
4863 case 1: /* D6 vline */
4864 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4865 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4866 DRM_DEBUG("IH: D6 vline\n");
4867 }
4868 break;
4869 default:
4870 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4871 break;
4872 }
4873 break;
4874 case 42: /* HPD hotplug */
4875 switch (src_data) {
4876 case 0:
4877 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4878 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4879 queue_hotplug = true;
4880 DRM_DEBUG("IH: HPD1\n");
4881 }
4882 break;
4883 case 1:
4884 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4885 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4886 queue_hotplug = true;
4887 DRM_DEBUG("IH: HPD2\n");
4888 }
4889 break;
4890 case 2:
4891 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4892 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4893 queue_hotplug = true;
4894 DRM_DEBUG("IH: HPD3\n");
4895 }
4896 break;
4897 case 3:
4898 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4899 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4900 queue_hotplug = true;
4901 DRM_DEBUG("IH: HPD4\n");
4902 }
4903 break;
4904 case 4:
4905 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4906 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4907 queue_hotplug = true;
4908 DRM_DEBUG("IH: HPD5\n");
4909 }
4910 break;
4911 case 5:
4912 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4913 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4914 queue_hotplug = true;
4915 DRM_DEBUG("IH: HPD6\n");
4916 }
4917 break;
4918 default:
4919 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4920 break;
4921 }
4922 break;
ae133a11
CK		 4923 case 146:
4924 case 147:
4925 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4926 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4927 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4928 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4929 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4930 /* reset addr and status */
4931 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4932 break;
25a857fb
AD		 4933 case 176: /* RINGID0 CP_INT */
4934 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4935 break;
4936 case 177: /* RINGID1 CP_INT */
4937 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4938 break;
4939 case 178: /* RINGID2 CP_INT */
4940 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4941 break;
4942 case 181: /* CP EOP event */
4943 DRM_DEBUG("IH: CP EOP\n");
4944 switch (ring_id) {
4945 case 0:
4946 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4947 break;
4948 case 1:
4949 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4950 break;
4951 case 2:
4952 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4953 break;
4954 }
4955 break;
8c5fd7ef
AD		 4956 case 224: /* DMA trap event */
4957 DRM_DEBUG("IH: DMA trap\n");
4958 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4959 break;
25a857fb
AD		 4960 case 233: /* GUI IDLE */
4961 DRM_DEBUG("IH: GUI idle\n");
25a857fb 4962 break;
8c5fd7ef
AD		 4963 case 244: /* DMA trap event */
4964 DRM_DEBUG("IH: DMA1 trap\n");
4965 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4966 break;
25a857fb
AD		 4967 default:
4968 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4969 break;
4970 }
4971
4972 /* wptr/rptr are in bytes! */
4973 rptr += 16;
4974 rptr &= rdev->ih.ptr_mask;
4975 }
25a857fb
AD		 4976 if (queue_hotplug)
4977 schedule_work(&rdev->hotplug_work);
4978 rdev->ih.rptr = rptr;
4979 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 4980 atomic_set(&rdev->ih.lock, 0);
4981
4982 /* make sure wptr hasn't changed while processing */
4983 wptr = si_get_ih_wptr(rdev);
4984 if (wptr != rptr)
4985 goto restart_ih;
4986
25a857fb
AD		 4987 return IRQ_HANDLED;
4988}
4989
8c5fd7ef
AD		 4990/**
4991 * si_copy_dma - copy pages using the DMA engine
4992 *
4993 * @rdev: radeon_device pointer
4994 * @src_offset: src GPU address
4995 * @dst_offset: dst GPU address
4996 * @num_gpu_pages: number of GPU pages to xfer
4997 * @fence: radeon fence object
4998 *
4999 * Copy GPU paging using the DMA engine (SI).
5000 * Used by the radeon ttm implementation to move pages if
5001 * registered as the asic copy callback.
5002 */
5003int si_copy_dma(struct radeon_device *rdev,
5004 uint64_t src_offset, uint64_t dst_offset,
5005 unsigned num_gpu_pages,
5006 struct radeon_fence **fence)
5007{
5008 struct radeon_semaphore *sem = NULL;
5009 int ring_index = rdev->asic->copy.dma_ring_index;
5010 struct radeon_ring *ring = &rdev->ring[ring_index];
5011 u32 size_in_bytes, cur_size_in_bytes;
5012 int i, num_loops;
5013 int r = 0;
5014
5015 r = radeon_semaphore_create(rdev, &sem);
5016 if (r) {
5017 DRM_ERROR("radeon: moving bo (%d).\n", r);
5018 return r;
5019 }
5020
5021 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
5022 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
5023 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5024 if (r) {
5025 DRM_ERROR("radeon: moving bo (%d).\n", r);
5026 radeon_semaphore_free(rdev, &sem, NULL);
5027 return r;
5028 }
5029
5030 if (radeon_fence_need_sync(*fence, ring->idx)) {
5031 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5032 ring->idx);
5033 radeon_fence_note_sync(*fence, ring->idx);
5034 } else {
5035 radeon_semaphore_free(rdev, &sem, NULL);
5036 }
5037
5038 for (i = 0; i < num_loops; i++) {
5039 cur_size_in_bytes = size_in_bytes;
5040 if (cur_size_in_bytes > 0xFFFFF)
5041 cur_size_in_bytes = 0xFFFFF;
5042 size_in_bytes -= cur_size_in_bytes;
5043 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
5044 radeon_ring_write(ring, dst_offset & 0xffffffff);
5045 radeon_ring_write(ring, src_offset & 0xffffffff);
5046 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5047 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5048 src_offset += cur_size_in_bytes;
5049 dst_offset += cur_size_in_bytes;
5050 }
5051
5052 r = radeon_fence_emit(rdev, fence, ring->idx);
5053 if (r) {
5054 radeon_ring_unlock_undo(rdev, ring);
5055 return r;
5056 }
5057
5058 radeon_ring_unlock_commit(rdev, ring);
5059 radeon_semaphore_free(rdev, &sem, *fence);
5060
5061 return r;
5062}
5063
9b136d51
AD		 5064/*
5065 * startup/shutdown callbacks
5066 */
5067static int si_startup(struct radeon_device *rdev)
5068{
5069 struct radeon_ring *ring;
5070 int r;
5071
5072 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5073 !rdev->rlc_fw || !rdev->mc_fw) {
5074 r = si_init_microcode(rdev);
5075 if (r) {
5076 DRM_ERROR("Failed to load firmware!\n");
5077 return r;
5078 }
5079 }
5080
5081 r = si_mc_load_microcode(rdev);
5082 if (r) {
5083 DRM_ERROR("Failed to load MC firmware!\n");
5084 return r;
5085 }
5086
5087 r = r600_vram_scratch_init(rdev);
5088 if (r)
5089 return r;
5090
5091 si_mc_program(rdev);
5092 r = si_pcie_gart_enable(rdev);
5093 if (r)
5094 return r;
5095 si_gpu_init(rdev);
5096
9b136d51
AD		 5097 /* allocate rlc buffers */
5098 r = si_rlc_init(rdev);
5099 if (r) {
5100 DRM_ERROR("Failed to init rlc BOs!\n");
5101 return r;
5102 }
5103
5104 /* allocate wb buffer */
5105 r = radeon_wb_init(rdev);
5106 if (r)
5107 return r;
5108
5109 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5110 if (r) {
5111 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5112 return r;
5113 }
5114
5115 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5116 if (r) {
5117 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5118 return r;
5119 }
5120
5121 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5122 if (r) {
5123 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5124 return r;
5125 }
5126
8c5fd7ef
AD		 5127 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5128 if (r) {
5129 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5130 return r;
5131 }
5132
5133 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5134 if (r) {
5135 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5136 return r;
5137 }
5138
f2ba57b5
CK		 5139 r = rv770_uvd_resume(rdev);
5140 if (!r) {
5141 r = radeon_fence_driver_start_ring(rdev,
5142 R600_RING_TYPE_UVD_INDEX);
5143 if (r)
5144 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5145 }
5146 if (r)
5147 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5148
9b136d51
AD		 5149 /* Enable IRQ */
5150 r = si_irq_init(rdev);
5151 if (r) {
5152 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5153 radeon_irq_kms_fini(rdev);
5154 return r;
5155 }
5156 si_irq_set(rdev);
5157
5158 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5159 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5160 CP_RB0_RPTR, CP_RB0_WPTR,
5161 0, 0xfffff, RADEON_CP_PACKET2);
5162 if (r)
5163 return r;
5164
5165 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5166 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
5167 CP_RB1_RPTR, CP_RB1_WPTR,
5168 0, 0xfffff, RADEON_CP_PACKET2);
5169 if (r)
5170 return r;
5171
5172 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5173 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
5174 CP_RB2_RPTR, CP_RB2_WPTR,
5175 0, 0xfffff, RADEON_CP_PACKET2);
5176 if (r)
5177 return r;
5178
8c5fd7ef
AD		 5179 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5180 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5181 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
5182 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
5183 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5184 if (r)
5185 return r;
5186
5187 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5188 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
5189 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
5190 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
5191 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5192 if (r)
5193 return r;
5194
9b136d51
AD		 5195 r = si_cp_load_microcode(rdev);
5196 if (r)
5197 return r;
5198 r = si_cp_resume(rdev);
5199 if (r)
5200 return r;
5201
8c5fd7ef
AD		 5202 r = cayman_dma_resume(rdev);
5203 if (r)
5204 return r;
5205
f2ba57b5
CK		 5206 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5207 if (ring->ring_size) {
5208 r = radeon_ring_init(rdev, ring, ring->ring_size,
5209 R600_WB_UVD_RPTR_OFFSET,
5210 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5211 0, 0xfffff, RADEON_CP_PACKET2);
5212 if (!r)
5213 r = r600_uvd_init(rdev);
5214 if (r)
5215 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
5216 }
5217
2898c348
CK		 5218 r = radeon_ib_pool_init(rdev);
5219 if (r) {
5220 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 5221 return r;
2898c348 5222 }
9b136d51 5223
c6105f24
CK		 5224 r = radeon_vm_manager_init(rdev);
5225 if (r) {
5226 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 5227 return r;
c6105f24 5228 }
9b136d51
AD		 5229
5230 return 0;
5231}
5232
5233int si_resume(struct radeon_device *rdev)
5234{
5235 int r;
5236
5237 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5238 * posting will perform necessary task to bring back GPU into good
5239 * shape.
5240 */
5241 /* post card */
5242 atom_asic_init(rdev->mode_info.atom_context);
5243
205996c0
AD		 5244 /* init golden registers */
5245 si_init_golden_registers(rdev);
5246
9b136d51
AD		 5247 rdev->accel_working = true;
5248 r = si_startup(rdev);
5249 if (r) {
5250 DRM_ERROR("si startup failed on resume\n");
5251 rdev->accel_working = false;
5252 return r;
5253 }
5254
5255 return r;
5256
5257}
5258
5259int si_suspend(struct radeon_device *rdev)
5260{
fa3daf9a 5261 radeon_vm_manager_fini(rdev);
9b136d51 5262 si_cp_enable(rdev, false);
8c5fd7ef 5263 cayman_dma_stop(rdev);
f2ba57b5
CK		 5264 r600_uvd_rbc_stop(rdev);
5265 radeon_uvd_suspend(rdev);
9b136d51
AD		 5266 si_irq_suspend(rdev);
5267 radeon_wb_disable(rdev);
5268 si_pcie_gart_disable(rdev);
5269 return 0;
5270}
5271
5272/* Plan is to move initialization in that function and use
5273 * helper function so that radeon_device_init pretty much
5274 * do nothing more than calling asic specific function. This
5275 * should also allow to remove a bunch of callback function
5276 * like vram_info.
5277 */
5278int si_init(struct radeon_device *rdev)
5279{
5280 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5281 int r;
5282
9b136d51
AD		 5283 /* Read BIOS */
5284 if (!radeon_get_bios(rdev)) {
5285 if (ASIC_IS_AVIVO(rdev))
5286 return -EINVAL;
5287 }
5288 /* Must be an ATOMBIOS */
5289 if (!rdev->is_atom_bios) {
5290 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
5291 return -EINVAL;
5292 }
5293 r = radeon_atombios_init(rdev);
5294 if (r)
5295 return r;
5296
5297 /* Post card if necessary */
5298 if (!radeon_card_posted(rdev)) {
5299 if (!rdev->bios) {
5300 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5301 return -EINVAL;
5302 }
5303 DRM_INFO("GPU not posted. posting now...\n");
5304 atom_asic_init(rdev->mode_info.atom_context);
5305 }
205996c0
AD		 5306 /* init golden registers */
5307 si_init_golden_registers(rdev);
9b136d51
AD		 5308 /* Initialize scratch registers */
5309 si_scratch_init(rdev);
5310 /* Initialize surface registers */
5311 radeon_surface_init(rdev);
5312 /* Initialize clocks */
5313 radeon_get_clock_info(rdev->ddev);
5314
5315 /* Fence driver */
5316 r = radeon_fence_driver_init(rdev);
5317 if (r)
5318 return r;
5319
5320 /* initialize memory controller */
5321 r = si_mc_init(rdev);
5322 if (r)
5323 return r;
5324 /* Memory manager */
5325 r = radeon_bo_init(rdev);
5326 if (r)
5327 return r;
5328
5329 r = radeon_irq_kms_init(rdev);
5330 if (r)
5331 return r;
5332
5333 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5334 ring->ring_obj = NULL;
5335 r600_ring_init(rdev, ring, 1024 * 1024);
5336
5337 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5338 ring->ring_obj = NULL;
5339 r600_ring_init(rdev, ring, 1024 * 1024);
5340
5341 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5342 ring->ring_obj = NULL;
5343 r600_ring_init(rdev, ring, 1024 * 1024);
5344
8c5fd7ef
AD		 5345 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5346 ring->ring_obj = NULL;
5347 r600_ring_init(rdev, ring, 64 * 1024);
5348
5349 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5350 ring->ring_obj = NULL;
5351 r600_ring_init(rdev, ring, 64 * 1024);
5352
f2ba57b5
CK		 5353 r = radeon_uvd_init(rdev);
5354 if (!r) {
5355 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5356 ring->ring_obj = NULL;
5357 r600_ring_init(rdev, ring, 4096);
5358 }
5359
9b136d51
AD		 5360 rdev->ih.ring_obj = NULL;
5361 r600_ih_ring_init(rdev, 64 * 1024);
5362
5363 r = r600_pcie_gart_init(rdev);
5364 if (r)
5365 return r;
5366
9b136d51 5367 rdev->accel_working = true;
9b136d51
AD		 5368 r = si_startup(rdev);
5369 if (r) {
5370 dev_err(rdev->dev, "disabling GPU acceleration\n");
5371 si_cp_fini(rdev);
8c5fd7ef 5372 cayman_dma_fini(rdev);
9b136d51
AD		 5373 si_irq_fini(rdev);
5374 si_rlc_fini(rdev);
5375 radeon_wb_fini(rdev);
2898c348 5376 radeon_ib_pool_fini(rdev);
9b136d51
AD		 5377 radeon_vm_manager_fini(rdev);
5378 radeon_irq_kms_fini(rdev);
5379 si_pcie_gart_fini(rdev);
5380 rdev->accel_working = false;
5381 }
5382
5383 /* Don't start up if the MC ucode is missing.
5384 * The default clocks and voltages before the MC ucode
5385 * is loaded are not suffient for advanced operations.
5386 */
5387 if (!rdev->mc_fw) {
5388 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5389 return -EINVAL;
5390 }
5391
5392 return 0;
5393}
5394
5395void si_fini(struct radeon_device *rdev)
5396{
9b136d51 5397 si_cp_fini(rdev);
8c5fd7ef 5398 cayman_dma_fini(rdev);
9b136d51
AD		 5399 si_irq_fini(rdev);
5400 si_rlc_fini(rdev);
5401 radeon_wb_fini(rdev);
5402 radeon_vm_manager_fini(rdev);
2898c348 5403 radeon_ib_pool_fini(rdev);
9b136d51 5404 radeon_irq_kms_fini(rdev);
f2ba57b5 5405 radeon_uvd_fini(rdev);
9b136d51
AD		 5406 si_pcie_gart_fini(rdev);
5407 r600_vram_scratch_fini(rdev);
5408 radeon_gem_fini(rdev);
9b136d51
AD		 5409 radeon_fence_driver_fini(rdev);
5410 radeon_bo_fini(rdev);
5411 radeon_atombios_fini(rdev);
5412 kfree(rdev->bios);
5413 rdev->bios = NULL;
5414}
5415
6759a0a7 5416/**
d0418894 5417 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 5418 *
5419 * @rdev: radeon_device pointer
5420 *
5421 * Fetches a GPU clock counter snapshot (SI).
5422 * Returns the 64 bit clock counter snapshot.
5423 */
d0418894 5424uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 5425{
5426 uint64_t clock;
5427
5428 mutex_lock(&rdev->gpu_clock_mutex);
5429 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5430 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
5431 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5432 mutex_unlock(&rdev->gpu_clock_mutex);
5433 return clock;
5434}
2539eb02 5435
2539eb02
CK		 5436int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
5437{
facd112d 5438 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 5439 int r;
5440
4ed10835
CK		 5441 /* bypass vclk and dclk with bclk */
5442 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5443 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
5444 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5445
5446 /* put PLL in bypass mode */
5447 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
5448
5449 if (!vclk || !dclk) {
5450 /* keep the Bypass mode, put PLL to sleep */
5451 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5452 return 0;
5453 }
5454
facd112d
CK		 5455 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
5456 16384, 0x03FFFFFF, 0, 128, 5,
5457 &fb_div, &vclk_div, &dclk_div);
5458 if (r)
5459 return r;
2539eb02
CK		 5460
5461 /* set RESET_ANTI_MUX to 0 */
5462 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
5463
5464 /* set VCO_MODE to 1 */
5465 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
5466
5467 /* toggle UPLL_SLEEP to 1 then back to 0 */
5468 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5469 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
5470
5471 /* deassert UPLL_RESET */
5472 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5473
5474 mdelay(1);
5475
facd112d 5476 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 5477 if (r)
5478 return r;
5479
5480 /* assert UPLL_RESET again */
5481 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
5482
5483 /* disable spread spectrum. */
5484 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
5485
5486 /* set feedback divider */
facd112d 5487 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 5488
5489 /* set ref divider to 0 */
5490 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
5491
facd112d 5492 if (fb_div < 307200)
2539eb02
CK		 5493 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
5494 else
5495 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
5496
5497 /* set PDIV_A and PDIV_B */
5498 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 5499 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 5500 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
5501
5502 /* give the PLL some time to settle */
5503 mdelay(15);
5504
5505 /* deassert PLL_RESET */
5506 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5507
5508 mdelay(15);
5509
5510 /* switch from bypass mode to normal mode */
5511 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
5512
facd112d 5513 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 5514 if (r)
5515 return r;
5516
5517 /* switch VCLK and DCLK selection */
5518 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5519 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
5520 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5521
5522 mdelay(100);
5523
5524 return 0;
5525}
Linux 6.x block layer and io_uring tree(s)