Commit | Line | Data |
---|---|---|
4b565ca5 | 1 | // SPDX-License-Identifier: GPL-2.0 |
e812744c | 2 | /* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */ |
4b565ca5 JC |
3 | |
4 | ||
5 | #include "msm_gem.h" | |
6 | #include "msm_mmu.h" | |
4241db42 | 7 | #include "msm_gpu_trace.h" |
4b565ca5 JC |
8 | #include "a6xx_gpu.h" |
9 | #include "a6xx_gmu.xml.h" | |
10 | ||
474dadb8 | 11 | #include <linux/bitfield.h> |
a2c3c0a5 | 12 | #include <linux/devfreq.h> |
c11fa120 | 13 | #include <linux/pm_domain.h> |
474dadb8 | 14 | #include <linux/soc/qcom/llcc-qcom.h> |
a2c3c0a5 | 15 | |
abccb9fe JC |
16 | #define GPU_PAS_ID 13 |
17 | ||
4b565ca5 JC |
18 | static inline bool _a6xx_check_idle(struct msm_gpu *gpu) |
19 | { | |
20 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
21 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
22 | ||
23 | /* Check that the GMU is idle */ | |
5a903a44 | 24 | if (!adreno_has_gmu_wrapper(adreno_gpu) && !a6xx_gmu_isidle(&a6xx_gpu->gmu)) |
4b565ca5 JC |
25 | return false; |
26 | ||
27 | /* Check tha the CX master is idle */ | |
28 | if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) & | |
29 | ~A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER) | |
30 | return false; | |
31 | ||
32 | return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) & | |
33 | A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT); | |
34 | } | |
35 | ||
991a2719 | 36 | static bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring) |
4b565ca5 JC |
37 | { |
38 | /* wait for CP to drain ringbuffer: */ | |
39 | if (!adreno_idle(gpu, ring)) | |
40 | return false; | |
41 | ||
42 | if (spin_until(_a6xx_check_idle(gpu))) { | |
43 | DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n", | |
44 | gpu->name, __builtin_return_address(0), | |
45 | gpu_read(gpu, REG_A6XX_RBBM_STATUS), | |
46 | gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS), | |
47 | gpu_read(gpu, REG_A6XX_CP_RB_RPTR), | |
48 | gpu_read(gpu, REG_A6XX_CP_RB_WPTR)); | |
49 | return false; | |
50 | } | |
51 | ||
52 | return true; | |
53 | } | |
54 | ||
0710a740 | 55 | static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) |
4b565ca5 | 56 | { |
d3a569fc JC |
57 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); |
58 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
4b565ca5 | 59 | |
d3a569fc JC |
60 | /* Expanded APRIV doesn't need to issue the WHERE_AM_I opcode */ |
61 | if (a6xx_gpu->has_whereami && !adreno_gpu->base.hw_apriv) { | |
d3a569fc JC |
62 | OUT_PKT7(ring, CP_WHERE_AM_I, 2); |
63 | OUT_RING(ring, lower_32_bits(shadowptr(a6xx_gpu, ring))); | |
64 | OUT_RING(ring, upper_32_bits(shadowptr(a6xx_gpu, ring))); | |
65 | } | |
0710a740 RC |
66 | } |
67 | ||
68 | static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring) | |
69 | { | |
70 | uint32_t wptr; | |
71 | unsigned long flags; | |
72 | ||
73 | update_shadow_rptr(gpu, ring); | |
d3a569fc | 74 | |
77c40603 | 75 | spin_lock_irqsave(&ring->preempt_lock, flags); |
4b565ca5 JC |
76 | |
77 | /* Copy the shadow to the actual register */ | |
78 | ring->cur = ring->next; | |
79 | ||
80 | /* Make sure to wrap wptr if we need to */ | |
81 | wptr = get_wptr(ring); | |
82 | ||
77c40603 | 83 | spin_unlock_irqrestore(&ring->preempt_lock, flags); |
4b565ca5 JC |
84 | |
85 | /* Make sure everything is posted before making a decision */ | |
86 | mb(); | |
87 | ||
88 | gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr); | |
89 | } | |
90 | ||
56869210 JC |
91 | static void get_stats_counter(struct msm_ringbuffer *ring, u32 counter, |
92 | u64 iova) | |
93 | { | |
94 | OUT_PKT7(ring, CP_REG_TO_MEM, 3); | |
b5e02e11 RC |
95 | OUT_RING(ring, CP_REG_TO_MEM_0_REG(counter) | |
96 | CP_REG_TO_MEM_0_CNT(2) | | |
97 | CP_REG_TO_MEM_0_64B); | |
56869210 JC |
98 | OUT_RING(ring, lower_32_bits(iova)); |
99 | OUT_RING(ring, upper_32_bits(iova)); | |
100 | } | |
101 | ||
84c31ee1 JC |
102 | static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu, |
103 | struct msm_ringbuffer *ring, struct msm_file_private *ctx) | |
104 | { | |
5f9ffe89 | 105 | bool sysprof = refcount_read(&a6xx_gpu->base.base.sysprof_active) > 1; |
af66706a | 106 | struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; |
84c31ee1 JC |
107 | phys_addr_t ttbr; |
108 | u32 asid; | |
109 | u64 memptr = rbmemptr(ring, ttbr0); | |
110 | ||
1d054c9b | 111 | if (ctx->seqno == a6xx_gpu->base.base.cur_ctx_seqno) |
84c31ee1 JC |
112 | return; |
113 | ||
114 | if (msm_iommu_pagetable_params(ctx->aspace->mmu, &ttbr, &asid)) | |
115 | return; | |
116 | ||
5f9ffe89 | 117 | if (!sysprof) { |
af66706a KD |
118 | if (!adreno_is_a7xx(adreno_gpu)) { |
119 | /* Turn off protected mode to write to special registers */ | |
120 | OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); | |
121 | OUT_RING(ring, 0); | |
122 | } | |
5f9ffe89 RC |
123 | |
124 | OUT_PKT4(ring, REG_A6XX_RBBM_PERFCTR_SRAM_INIT_CMD, 1); | |
125 | OUT_RING(ring, 1); | |
126 | } | |
127 | ||
84c31ee1 JC |
128 | /* Execute the table update */ |
129 | OUT_PKT7(ring, CP_SMMU_TABLE_UPDATE, 4); | |
130 | OUT_RING(ring, CP_SMMU_TABLE_UPDATE_0_TTBR0_LO(lower_32_bits(ttbr))); | |
131 | ||
132 | OUT_RING(ring, | |
133 | CP_SMMU_TABLE_UPDATE_1_TTBR0_HI(upper_32_bits(ttbr)) | | |
134 | CP_SMMU_TABLE_UPDATE_1_ASID(asid)); | |
135 | OUT_RING(ring, CP_SMMU_TABLE_UPDATE_2_CONTEXTIDR(0)); | |
136 | OUT_RING(ring, CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK(0)); | |
137 | ||
138 | /* | |
139 | * Write the new TTBR0 to the memstore. This is good for debugging. | |
140 | */ | |
141 | OUT_PKT7(ring, CP_MEM_WRITE, 4); | |
142 | OUT_RING(ring, CP_MEM_WRITE_0_ADDR_LO(lower_32_bits(memptr))); | |
143 | OUT_RING(ring, CP_MEM_WRITE_1_ADDR_HI(upper_32_bits(memptr))); | |
144 | OUT_RING(ring, lower_32_bits(ttbr)); | |
145 | OUT_RING(ring, (asid << 16) | upper_32_bits(ttbr)); | |
146 | ||
af66706a KD |
147 | /* |
148 | * Sync both threads after switching pagetables and enable BR only | |
149 | * to make sure BV doesn't race ahead while BR is still switching | |
150 | * pagetables. | |
151 | */ | |
152 | if (adreno_is_a7xx(&a6xx_gpu->base)) { | |
153 | OUT_PKT7(ring, CP_THREAD_CONTROL, 1); | |
154 | OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BR); | |
155 | } | |
156 | ||
84c31ee1 JC |
157 | /* |
158 | * And finally, trigger a uche flush to be sure there isn't anything | |
159 | * lingering in that part of the GPU | |
160 | */ | |
161 | ||
162 | OUT_PKT7(ring, CP_EVENT_WRITE, 1); | |
80059b87 | 163 | OUT_RING(ring, CACHE_INVALIDATE); |
5f9ffe89 RC |
164 | |
165 | if (!sysprof) { | |
166 | /* | |
167 | * Wait for SRAM clear after the pgtable update, so the | |
168 | * two can happen in parallel: | |
169 | */ | |
170 | OUT_PKT7(ring, CP_WAIT_REG_MEM, 6); | |
171 | OUT_RING(ring, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ)); | |
172 | OUT_RING(ring, CP_WAIT_REG_MEM_1_POLL_ADDR_LO( | |
173 | REG_A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS)); | |
174 | OUT_RING(ring, CP_WAIT_REG_MEM_2_POLL_ADDR_HI(0)); | |
175 | OUT_RING(ring, CP_WAIT_REG_MEM_3_REF(0x1)); | |
176 | OUT_RING(ring, CP_WAIT_REG_MEM_4_MASK(0x1)); | |
177 | OUT_RING(ring, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(0)); | |
178 | ||
af66706a KD |
179 | if (!adreno_is_a7xx(adreno_gpu)) { |
180 | /* Re-enable protected mode: */ | |
181 | OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); | |
182 | OUT_RING(ring, 1); | |
183 | } | |
5f9ffe89 | 184 | } |
84c31ee1 JC |
185 | } |
186 | ||
15eb9ad0 | 187 | static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) |
4b565ca5 | 188 | { |
56869210 | 189 | unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT; |
4241db42 JC |
190 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); |
191 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
4b565ca5 | 192 | struct msm_ringbuffer *ring = submit->ring; |
0710a740 | 193 | unsigned int i, ibs = 0; |
4b565ca5 | 194 | |
84c31ee1 JC |
195 | a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx); |
196 | ||
cc4c26d4 | 197 | get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0), |
56869210 JC |
198 | rbmemptr_stats(ring, index, cpcycles_start)); |
199 | ||
200 | /* | |
201 | * For PM4 the GMU register offsets are calculated from the base of the | |
202 | * GPU registers so we need to add 0x1a800 to the register value on A630 | |
203 | * to get the right value from PM4. | |
204 | */ | |
f73343fa | 205 | get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER, |
56869210 JC |
206 | rbmemptr_stats(ring, index, alwayson_start)); |
207 | ||
4b565ca5 JC |
208 | /* Invalidate CCU depth and color */ |
209 | OUT_PKT7(ring, CP_EVENT_WRITE, 1); | |
b5e02e11 | 210 | OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(PC_CCU_INVALIDATE_DEPTH)); |
4b565ca5 JC |
211 | |
212 | OUT_PKT7(ring, CP_EVENT_WRITE, 1); | |
b5e02e11 | 213 | OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(PC_CCU_INVALIDATE_COLOR)); |
4b565ca5 JC |
214 | |
215 | /* Submit the commands */ | |
216 | for (i = 0; i < submit->nr_cmds; i++) { | |
217 | switch (submit->cmd[i].type) { | |
218 | case MSM_SUBMIT_CMD_IB_TARGET_BUF: | |
219 | break; | |
220 | case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: | |
1d054c9b | 221 | if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno) |
4b565ca5 | 222 | break; |
df561f66 | 223 | fallthrough; |
4b565ca5 JC |
224 | case MSM_SUBMIT_CMD_BUF: |
225 | OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3); | |
226 | OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); | |
227 | OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); | |
228 | OUT_RING(ring, submit->cmd[i].size); | |
0710a740 | 229 | ibs++; |
4b565ca5 JC |
230 | break; |
231 | } | |
0710a740 RC |
232 | |
233 | /* | |
234 | * Periodically update shadow-wptr if needed, so that we | |
235 | * can see partial progress of submits with large # of | |
236 | * cmds.. otherwise we could needlessly stall waiting for | |
237 | * ringbuffer state, simply due to looking at a shadow | |
238 | * rptr value that has not been updated | |
239 | */ | |
240 | if ((ibs % 32) == 0) | |
241 | update_shadow_rptr(gpu, ring); | |
4b565ca5 JC |
242 | } |
243 | ||
cc4c26d4 | 244 | get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0), |
56869210 | 245 | rbmemptr_stats(ring, index, cpcycles_end)); |
f73343fa | 246 | get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER, |
56869210 JC |
247 | rbmemptr_stats(ring, index, alwayson_end)); |
248 | ||
4b565ca5 JC |
249 | /* Write the fence to the scratch register */ |
250 | OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1); | |
251 | OUT_RING(ring, submit->seqno); | |
252 | ||
253 | /* | |
254 | * Execute a CACHE_FLUSH_TS event. This will ensure that the | |
255 | * timestamp is written to the memory and then triggers the interrupt | |
256 | */ | |
257 | OUT_PKT7(ring, CP_EVENT_WRITE, 4); | |
b5e02e11 RC |
258 | OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) | |
259 | CP_EVENT_WRITE_0_IRQ); | |
4b565ca5 JC |
260 | OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence))); |
261 | OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence))); | |
262 | OUT_RING(ring, submit->seqno); | |
263 | ||
4241db42 | 264 | trace_msm_gpu_submit_flush(submit, |
f73343fa | 265 | gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER)); |
4241db42 | 266 | |
4b565ca5 JC |
267 | a6xx_flush(gpu, ring); |
268 | } | |
269 | ||
af66706a KD |
270 | static void a7xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) |
271 | { | |
272 | unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT; | |
273 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
274 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
275 | struct msm_ringbuffer *ring = submit->ring; | |
276 | unsigned int i, ibs = 0; | |
277 | ||
278 | /* | |
279 | * Toggle concurrent binning for pagetable switch and set the thread to | |
280 | * BR since only it can execute the pagetable switch packets. | |
281 | */ | |
282 | OUT_PKT7(ring, CP_THREAD_CONTROL, 1); | |
283 | OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BR); | |
284 | ||
285 | a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx); | |
286 | ||
287 | get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0), | |
288 | rbmemptr_stats(ring, index, cpcycles_start)); | |
289 | get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER, | |
290 | rbmemptr_stats(ring, index, alwayson_start)); | |
291 | ||
292 | OUT_PKT7(ring, CP_THREAD_CONTROL, 1); | |
293 | OUT_RING(ring, CP_SET_THREAD_BOTH); | |
294 | ||
295 | OUT_PKT7(ring, CP_SET_MARKER, 1); | |
296 | OUT_RING(ring, 0x101); /* IFPC disable */ | |
297 | ||
298 | OUT_PKT7(ring, CP_SET_MARKER, 1); | |
299 | OUT_RING(ring, 0x00d); /* IB1LIST start */ | |
300 | ||
301 | /* Submit the commands */ | |
302 | for (i = 0; i < submit->nr_cmds; i++) { | |
303 | switch (submit->cmd[i].type) { | |
304 | case MSM_SUBMIT_CMD_IB_TARGET_BUF: | |
305 | break; | |
306 | case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: | |
307 | if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno) | |
308 | break; | |
309 | fallthrough; | |
310 | case MSM_SUBMIT_CMD_BUF: | |
311 | OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3); | |
312 | OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); | |
313 | OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); | |
314 | OUT_RING(ring, submit->cmd[i].size); | |
315 | ibs++; | |
316 | break; | |
317 | } | |
318 | ||
319 | /* | |
320 | * Periodically update shadow-wptr if needed, so that we | |
321 | * can see partial progress of submits with large # of | |
322 | * cmds.. otherwise we could needlessly stall waiting for | |
323 | * ringbuffer state, simply due to looking at a shadow | |
324 | * rptr value that has not been updated | |
325 | */ | |
326 | if ((ibs % 32) == 0) | |
327 | update_shadow_rptr(gpu, ring); | |
328 | } | |
329 | ||
330 | OUT_PKT7(ring, CP_SET_MARKER, 1); | |
331 | OUT_RING(ring, 0x00e); /* IB1LIST end */ | |
332 | ||
333 | get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0), | |
334 | rbmemptr_stats(ring, index, cpcycles_end)); | |
335 | get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER, | |
336 | rbmemptr_stats(ring, index, alwayson_end)); | |
337 | ||
338 | /* Write the fence to the scratch register */ | |
339 | OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1); | |
340 | OUT_RING(ring, submit->seqno); | |
341 | ||
342 | OUT_PKT7(ring, CP_THREAD_CONTROL, 1); | |
343 | OUT_RING(ring, CP_SET_THREAD_BR); | |
344 | ||
345 | OUT_PKT7(ring, CP_EVENT_WRITE, 1); | |
346 | OUT_RING(ring, CCU_INVALIDATE_DEPTH); | |
347 | ||
348 | OUT_PKT7(ring, CP_EVENT_WRITE, 1); | |
349 | OUT_RING(ring, CCU_INVALIDATE_COLOR); | |
350 | ||
351 | OUT_PKT7(ring, CP_THREAD_CONTROL, 1); | |
352 | OUT_RING(ring, CP_SET_THREAD_BV); | |
353 | ||
354 | /* | |
355 | * Make sure the timestamp is committed once BV pipe is | |
356 | * completely done with this submission. | |
357 | */ | |
358 | OUT_PKT7(ring, CP_EVENT_WRITE, 4); | |
359 | OUT_RING(ring, CACHE_CLEAN | BIT(27)); | |
360 | OUT_RING(ring, lower_32_bits(rbmemptr(ring, bv_fence))); | |
361 | OUT_RING(ring, upper_32_bits(rbmemptr(ring, bv_fence))); | |
362 | OUT_RING(ring, submit->seqno); | |
363 | ||
364 | OUT_PKT7(ring, CP_THREAD_CONTROL, 1); | |
365 | OUT_RING(ring, CP_SET_THREAD_BR); | |
366 | ||
367 | /* | |
368 | * This makes sure that BR doesn't race ahead and commit | |
369 | * timestamp to memstore while BV is still processing | |
370 | * this submission. | |
371 | */ | |
372 | OUT_PKT7(ring, CP_WAIT_TIMESTAMP, 4); | |
373 | OUT_RING(ring, 0); | |
374 | OUT_RING(ring, lower_32_bits(rbmemptr(ring, bv_fence))); | |
375 | OUT_RING(ring, upper_32_bits(rbmemptr(ring, bv_fence))); | |
376 | OUT_RING(ring, submit->seqno); | |
377 | ||
378 | /* write the ringbuffer timestamp */ | |
379 | OUT_PKT7(ring, CP_EVENT_WRITE, 4); | |
380 | OUT_RING(ring, CACHE_CLEAN | CP_EVENT_WRITE_0_IRQ | BIT(27)); | |
381 | OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence))); | |
382 | OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence))); | |
383 | OUT_RING(ring, submit->seqno); | |
384 | ||
385 | OUT_PKT7(ring, CP_THREAD_CONTROL, 1); | |
386 | OUT_RING(ring, CP_SET_THREAD_BOTH); | |
387 | ||
388 | OUT_PKT7(ring, CP_SET_MARKER, 1); | |
389 | OUT_RING(ring, 0x100); /* IFPC enable */ | |
390 | ||
391 | trace_msm_gpu_submit_flush(submit, | |
392 | gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER)); | |
393 | ||
394 | a6xx_flush(gpu, ring); | |
395 | } | |
396 | ||
e7fc9398 KD |
397 | const struct adreno_reglist a612_hwcg[] = { |
398 | {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222}, | |
399 | {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, | |
400 | {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000081}, | |
401 | {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf}, | |
402 | {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, | |
403 | {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, | |
404 | {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, | |
405 | {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, | |
406 | {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, | |
407 | {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, | |
408 | {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, | |
409 | {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, | |
410 | {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, | |
411 | {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, | |
412 | {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, | |
413 | {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, | |
414 | {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, | |
415 | {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01202222}, | |
416 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, | |
417 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00}, | |
418 | {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05522022}, | |
419 | {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, | |
420 | {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, | |
421 | {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, | |
422 | {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, | |
423 | {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, | |
424 | {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, | |
425 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, | |
426 | {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, | |
427 | {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, | |
428 | {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, | |
429 | {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, | |
430 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, | |
431 | {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, | |
432 | {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, | |
433 | {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, | |
434 | {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, | |
435 | {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, | |
436 | {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, | |
437 | {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, | |
438 | {REG_A6XX_RBBM_ISDB_CNT, 0x00000182}, | |
439 | {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, | |
440 | {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000}, | |
441 | {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, | |
442 | {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, | |
443 | {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, | |
444 | {}, | |
445 | }; | |
446 | ||
b7616b5c KD |
447 | /* For a615 family (a615, a616, a618 and a619) */ |
448 | const struct adreno_reglist a615_hwcg[] = { | |
449 | {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, | |
450 | {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, | |
451 | {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, | |
452 | {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, | |
453 | {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, | |
454 | {REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222}, | |
455 | {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, | |
456 | {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, | |
457 | {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, | |
458 | {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222}, | |
459 | {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, | |
460 | {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222}, | |
461 | {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, | |
462 | {REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, | |
463 | {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, | |
464 | {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, | |
465 | {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, | |
466 | {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777}, | |
467 | {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, | |
468 | {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777}, | |
469 | {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, | |
470 | {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, | |
471 | {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, | |
472 | {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, | |
473 | {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, | |
474 | {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111}, | |
475 | {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, | |
476 | {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111}, | |
477 | {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, | |
478 | {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, | |
479 | {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, | |
480 | {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, | |
481 | {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, | |
482 | {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, | |
483 | {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, | |
484 | {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222}, | |
485 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002020}, | |
486 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220}, | |
487 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220}, | |
488 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220}, | |
489 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, | |
490 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040F00}, | |
491 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040F00}, | |
492 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040F00}, | |
493 | {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022}, | |
494 | {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, | |
495 | {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, | |
496 | {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, | |
497 | {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, | |
498 | {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, | |
499 | {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, | |
500 | {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, | |
501 | {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, | |
502 | {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, | |
503 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, | |
504 | {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, | |
505 | {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, | |
506 | {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, | |
507 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, | |
508 | {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, | |
509 | {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, | |
510 | {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, | |
511 | {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, | |
512 | {}, | |
513 | }; | |
514 | ||
b1c53a2a | 515 | const struct adreno_reglist a630_hwcg[] = { |
4b565ca5 JC |
516 | {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222}, |
517 | {REG_A6XX_RBBM_CLOCK_CNTL_SP1, 0x22222222}, | |
518 | {REG_A6XX_RBBM_CLOCK_CNTL_SP2, 0x22222222}, | |
519 | {REG_A6XX_RBBM_CLOCK_CNTL_SP3, 0x22222222}, | |
520 | {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220}, | |
521 | {REG_A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220}, | |
522 | {REG_A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220}, | |
523 | {REG_A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220}, | |
524 | {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, | |
525 | {REG_A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, | |
526 | {REG_A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, | |
527 | {REG_A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, | |
528 | {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf}, | |
529 | {REG_A6XX_RBBM_CLOCK_HYST_SP1, 0x0000f3cf}, | |
530 | {REG_A6XX_RBBM_CLOCK_HYST_SP2, 0x0000f3cf}, | |
531 | {REG_A6XX_RBBM_CLOCK_HYST_SP3, 0x0000f3cf}, | |
532 | {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, | |
533 | {REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222}, | |
534 | {REG_A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222}, | |
535 | {REG_A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222}, | |
536 | {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, | |
537 | {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, | |
538 | {REG_A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, | |
539 | {REG_A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, | |
540 | {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, | |
541 | {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222}, | |
542 | {REG_A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222}, | |
543 | {REG_A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222}, | |
544 | {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, | |
545 | {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222}, | |
546 | {REG_A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222}, | |
547 | {REG_A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222}, | |
548 | {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, | |
549 | {REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, | |
550 | {REG_A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, | |
551 | {REG_A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, | |
552 | {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, | |
553 | {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, | |
554 | {REG_A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, | |
555 | {REG_A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, | |
556 | {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, | |
557 | {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777}, | |
558 | {REG_A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777}, | |
559 | {REG_A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777}, | |
560 | {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, | |
561 | {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777}, | |
562 | {REG_A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777}, | |
563 | {REG_A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777}, | |
564 | {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, | |
565 | {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, | |
566 | {REG_A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, | |
567 | {REG_A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, | |
568 | {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, | |
569 | {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, | |
570 | {REG_A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, | |
571 | {REG_A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, | |
572 | {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, | |
573 | {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111}, | |
574 | {REG_A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111}, | |
575 | {REG_A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111}, | |
576 | {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, | |
577 | {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111}, | |
578 | {REG_A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111}, | |
579 | {REG_A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111}, | |
580 | {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, | |
581 | {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, | |
582 | {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, | |
583 | {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, | |
584 | {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, | |
585 | {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, | |
586 | {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, | |
587 | {REG_A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, | |
588 | {REG_A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, | |
589 | {REG_A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, | |
590 | {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222}, | |
591 | {REG_A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222}, | |
592 | {REG_A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222}, | |
593 | {REG_A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222}, | |
594 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, | |
595 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220}, | |
596 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220}, | |
597 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220}, | |
598 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00}, | |
599 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040f00}, | |
600 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040f00}, | |
601 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040f00}, | |
602 | {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022}, | |
603 | {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, | |
604 | {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, | |
605 | {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, | |
606 | {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, | |
607 | {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, | |
608 | {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, | |
609 | {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, | |
610 | {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, | |
611 | {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, | |
612 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, | |
613 | {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, | |
614 | {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, | |
615 | {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, | |
616 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, | |
617 | {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, | |
618 | {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, | |
619 | {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, | |
b1c53a2a JM |
620 | {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, |
621 | {}, | |
4b565ca5 JC |
622 | }; |
623 | ||
66ffb915 JM |
624 | const struct adreno_reglist a640_hwcg[] = { |
625 | {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, | |
626 | {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, | |
627 | {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, | |
628 | {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, | |
629 | {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, | |
630 | {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, | |
631 | {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, | |
632 | {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, | |
633 | {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, | |
634 | {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, | |
635 | {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, | |
636 | {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, | |
637 | {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, | |
638 | {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, | |
639 | {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, | |
640 | {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, | |
641 | {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, | |
642 | {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222}, | |
643 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, | |
644 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, | |
645 | {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05222022}, | |
646 | {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, | |
647 | {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, | |
648 | {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, | |
649 | {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, | |
650 | {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, | |
651 | {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, | |
652 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, | |
653 | {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, | |
654 | {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, | |
655 | {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, | |
656 | {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, | |
657 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, | |
658 | {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, | |
659 | {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, | |
660 | {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, | |
661 | {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, | |
662 | {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222}, | |
663 | {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111}, | |
664 | {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000}, | |
665 | {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, | |
666 | {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, | |
667 | {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, | |
668 | {REG_A6XX_RBBM_ISDB_CNT, 0x00000182}, | |
669 | {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, | |
670 | {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000}, | |
671 | {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, | |
672 | {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, | |
673 | {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, | |
674 | {}, | |
675 | }; | |
676 | ||
677 | const struct adreno_reglist a650_hwcg[] = { | |
678 | {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, | |
679 | {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, | |
680 | {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, | |
681 | {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, | |
682 | {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, | |
683 | {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, | |
684 | {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, | |
685 | {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, | |
686 | {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, | |
687 | {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, | |
688 | {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, | |
689 | {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, | |
690 | {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, | |
691 | {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, | |
692 | {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, | |
693 | {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, | |
694 | {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, | |
695 | {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222}, | |
696 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, | |
697 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, | |
698 | {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022}, | |
699 | {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, | |
700 | {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, | |
701 | {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, | |
702 | {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, | |
703 | {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, | |
704 | {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, | |
705 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, | |
706 | {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, | |
707 | {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, | |
708 | {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, | |
709 | {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, | |
710 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, | |
711 | {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, | |
712 | {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, | |
713 | {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, | |
714 | {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, | |
715 | {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222}, | |
716 | {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111}, | |
717 | {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000777}, | |
718 | {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, | |
719 | {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, | |
720 | {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, | |
721 | {REG_A6XX_RBBM_ISDB_CNT, 0x00000182}, | |
722 | {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, | |
723 | {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000}, | |
724 | {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, | |
725 | {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, | |
726 | {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, | |
727 | {}, | |
728 | }; | |
729 | ||
f6d62d09 JM |
730 | const struct adreno_reglist a660_hwcg[] = { |
731 | {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, | |
732 | {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, | |
733 | {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, | |
734 | {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, | |
735 | {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, | |
736 | {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, | |
737 | {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, | |
738 | {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, | |
739 | {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, | |
740 | {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, | |
741 | {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, | |
742 | {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, | |
743 | {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, | |
744 | {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, | |
745 | {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, | |
746 | {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, | |
747 | {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, | |
748 | {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222}, | |
749 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, | |
750 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, | |
751 | {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022}, | |
752 | {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, | |
753 | {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, | |
754 | {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, | |
755 | {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, | |
756 | {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, | |
757 | {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, | |
758 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, | |
759 | {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, | |
760 | {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, | |
761 | {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, | |
762 | {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, | |
763 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, | |
764 | {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, | |
765 | {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, | |
766 | {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, | |
767 | {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, | |
768 | {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222}, | |
769 | {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111}, | |
770 | {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000}, | |
771 | {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, | |
772 | {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, | |
773 | {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, | |
774 | {REG_A6XX_RBBM_ISDB_CNT, 0x00000182}, | |
775 | {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, | |
776 | {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000}, | |
777 | {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, | |
778 | {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, | |
779 | {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, | |
780 | {}, | |
781 | }; | |
782 | ||
5e7665b5 BA |
783 | const struct adreno_reglist a690_hwcg[] = { |
784 | {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, | |
785 | {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, | |
786 | {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, | |
787 | {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, | |
788 | {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, | |
789 | {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, | |
790 | {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, | |
791 | {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, | |
792 | {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, | |
793 | {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, | |
794 | {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, | |
795 | {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, | |
796 | {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, | |
797 | {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, | |
798 | {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, | |
799 | {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, | |
800 | {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, | |
801 | {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222}, | |
802 | {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, | |
803 | {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, | |
804 | {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022}, | |
805 | {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, | |
806 | {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, | |
807 | {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, | |
808 | {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, | |
809 | {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, | |
810 | {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, | |
811 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, | |
812 | {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, | |
813 | {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, | |
814 | {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, | |
815 | {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, | |
816 | {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, | |
817 | {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, | |
818 | {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, | |
819 | {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, | |
820 | {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, | |
821 | {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222}, | |
822 | {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111}, | |
823 | {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000}, | |
824 | {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, | |
825 | {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, | |
826 | {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, | |
827 | {REG_A6XX_RBBM_CLOCK_CNTL, 0x8AA8AA82}, | |
828 | {REG_A6XX_RBBM_ISDB_CNT, 0x00000182}, | |
829 | {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, | |
830 | {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000}, | |
831 | {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, | |
832 | {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, | |
833 | {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, | |
834 | {REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL, 0x20200}, | |
835 | {REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, 0x10111}, | |
836 | {REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL, 0x5555}, | |
837 | {} | |
838 | }; | |
839 | ||
18397519 KD |
840 | const struct adreno_reglist a702_hwcg[] = { |
841 | { REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222 }, | |
842 | { REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220 }, | |
843 | { REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000081 }, | |
844 | { REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf }, | |
845 | { REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222 }, | |
846 | { REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 }, | |
847 | { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 }, | |
848 | { REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222 }, | |
849 | { REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111 }, | |
850 | { REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111 }, | |
851 | { REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111 }, | |
852 | { REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111 }, | |
853 | { REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 }, | |
854 | { REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 }, | |
855 | { REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777 }, | |
856 | { REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777 }, | |
857 | { REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222 }, | |
858 | { REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01202222 }, | |
859 | { REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220 }, | |
860 | { REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00 }, | |
861 | { REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05522022 }, | |
862 | { REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555 }, | |
863 | { REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011 }, | |
864 | { REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044 }, | |
865 | { REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 }, | |
866 | { REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222 }, | |
867 | { REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222222 }, | |
868 | { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002 }, | |
869 | { REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222 }, | |
870 | { REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000 }, | |
871 | { REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222 }, | |
872 | { REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200 }, | |
873 | { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 }, | |
874 | { REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 }, | |
875 | { REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000 }, | |
876 | { REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004 }, | |
877 | { REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000 }, | |
878 | { REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222 }, | |
879 | { REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004 }, | |
880 | { REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002 }, | |
881 | { REG_A6XX_RBBM_ISDB_CNT, 0x00000182 }, | |
882 | { REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000 }, | |
883 | { REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000 }, | |
884 | { REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 }, | |
885 | { REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 }, | |
886 | { REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 }, | |
887 | { REG_A6XX_RBBM_CLOCK_CNTL_FCHE, 0x00000222 }, | |
888 | { REG_A6XX_RBBM_CLOCK_DELAY_FCHE, 0x00000000 }, | |
889 | { REG_A6XX_RBBM_CLOCK_HYST_FCHE, 0x00000000 }, | |
890 | { REG_A6XX_RBBM_CLOCK_CNTL_GLC, 0x00222222 }, | |
891 | { REG_A6XX_RBBM_CLOCK_DELAY_GLC, 0x00000000 }, | |
892 | { REG_A6XX_RBBM_CLOCK_HYST_GLC, 0x00000000 }, | |
893 | { REG_A6XX_RBBM_CLOCK_CNTL_MHUB, 0x00000002 }, | |
894 | { REG_A6XX_RBBM_CLOCK_DELAY_MHUB, 0x00000000 }, | |
895 | { REG_A6XX_RBBM_CLOCK_HYST_MHUB, 0x00000000 }, | |
896 | {} | |
897 | }; | |
898 | ||
9588d2f8 KD |
899 | const struct adreno_reglist a730_hwcg[] = { |
900 | { REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222 }, | |
901 | { REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022222 }, | |
902 | { REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf }, | |
903 | { REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080 }, | |
904 | { REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222220 }, | |
905 | { REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 }, | |
906 | { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 }, | |
907 | { REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00222222 }, | |
908 | { REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 }, | |
909 | { REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 }, | |
910 | { REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777 }, | |
911 | { REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777 }, | |
912 | { REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111 }, | |
913 | { REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111 }, | |
914 | { REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111 }, | |
915 | { REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111 }, | |
916 | { REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222 }, | |
917 | { REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004 }, | |
918 | { REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002 }, | |
919 | { REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222 }, | |
920 | { REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222 }, | |
921 | { REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220 }, | |
922 | { REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x44000f00 }, | |
923 | { REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022 }, | |
924 | { REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00555555 }, | |
925 | { REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011 }, | |
926 | { REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00440044 }, | |
927 | { REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 }, | |
928 | { REG_A7XX_RBBM_CLOCK_MODE2_GRAS, 0x00000222 }, | |
929 | { REG_A7XX_RBBM_CLOCK_MODE_BV_GRAS, 0x00222222 }, | |
930 | { REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222223 }, | |
931 | { REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222 }, | |
932 | { REG_A7XX_RBBM_CLOCK_MODE_BV_GPC, 0x00222222 }, | |
933 | { REG_A7XX_RBBM_CLOCK_MODE_BV_VFD, 0x00002222 }, | |
934 | { REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 }, | |
935 | { REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004 }, | |
936 | { REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000 }, | |
937 | { REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000 }, | |
938 | { REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200 }, | |
939 | { REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222 }, | |
940 | { REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222 }, | |
941 | { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 }, | |
942 | { REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000 }, | |
943 | { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002 }, | |
944 | { REG_A7XX_RBBM_CLOCK_MODE_BV_LRZ, 0x55555552 }, | |
945 | { REG_A7XX_RBBM_CLOCK_MODE_CP, 0x00000223 }, | |
946 | { REG_A6XX_RBBM_CLOCK_CNTL, 0x8aa8aa82 }, | |
947 | { REG_A6XX_RBBM_ISDB_CNT, 0x00000182 }, | |
948 | { REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000 }, | |
949 | { REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000 }, | |
950 | { REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 }, | |
951 | { REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 }, | |
952 | { REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 }, | |
953 | {}, | |
954 | }; | |
955 | ||
1f8c29e8 KD |
956 | const struct adreno_reglist a740_hwcg[] = { |
957 | { REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222 }, | |
958 | { REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x22022222 }, | |
959 | { REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x003cf3cf }, | |
960 | { REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080 }, | |
961 | { REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222220 }, | |
962 | { REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 }, | |
963 | { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 }, | |
964 | { REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00222222 }, | |
965 | { REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 }, | |
966 | { REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 }, | |
967 | { REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777 }, | |
968 | { REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777 }, | |
969 | { REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111 }, | |
970 | { REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111 }, | |
971 | { REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111 }, | |
972 | { REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111 }, | |
973 | { REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222 }, | |
974 | { REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x00222222 }, | |
975 | { REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000444 }, | |
976 | { REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000222 }, | |
977 | { REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222 }, | |
978 | { REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222 }, | |
979 | { REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220 }, | |
980 | { REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x44000f00 }, | |
981 | { REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022 }, | |
982 | { REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00555555 }, | |
983 | { REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011 }, | |
984 | { REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00440044 }, | |
985 | { REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 }, | |
986 | { REG_A7XX_RBBM_CLOCK_MODE2_GRAS, 0x00000222 }, | |
987 | { REG_A7XX_RBBM_CLOCK_MODE_BV_GRAS, 0x00222222 }, | |
988 | { REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222223 }, | |
989 | { REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00222222 }, | |
990 | { REG_A7XX_RBBM_CLOCK_MODE_BV_GPC, 0x00222222 }, | |
991 | { REG_A7XX_RBBM_CLOCK_MODE_BV_VFD, 0x00002222 }, | |
992 | { REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 }, | |
993 | { REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004 }, | |
994 | { REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000 }, | |
995 | { REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00000000 }, | |
996 | { REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200 }, | |
997 | { REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00000000 }, | |
998 | { REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222 }, | |
999 | { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 }, | |
1000 | { REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000 }, | |
1001 | { REG_A7XX_RBBM_CLOCK_MODE_BV_LRZ, 0x55555552 }, | |
1002 | { REG_A7XX_RBBM_CLOCK_HYST2_VFD, 0x00000000 }, | |
1003 | { REG_A7XX_RBBM_CLOCK_MODE_CP, 0x00000222 }, | |
1004 | { REG_A6XX_RBBM_CLOCK_CNTL, 0x8aa8aa82 }, | |
1005 | { REG_A6XX_RBBM_ISDB_CNT, 0x00000182 }, | |
1006 | { REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000 }, | |
1007 | { REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000 }, | |
1008 | { REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 }, | |
1009 | { REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 }, | |
1010 | { REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 }, | |
1011 | {}, | |
1012 | }; | |
1013 | ||
4b565ca5 JC |
1014 | static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state) |
1015 | { | |
1016 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
1017 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
1018 | struct a6xx_gmu *gmu = &a6xx_gpu->gmu; | |
b1c53a2a | 1019 | const struct adreno_reglist *reg; |
4b565ca5 | 1020 | unsigned int i; |
1f8c29e8 | 1021 | u32 val, clock_cntl_on, cgc_mode; |
4b565ca5 | 1022 | |
d2bcca0c | 1023 | if (!(adreno_gpu->info->hwcg || adreno_is_a7xx(adreno_gpu))) |
b1c53a2a JM |
1024 | return; |
1025 | ||
66ffb915 JM |
1026 | if (adreno_is_a630(adreno_gpu)) |
1027 | clock_cntl_on = 0x8aa8aa02; | |
e7fc9398 KD |
1028 | else if (adreno_is_a610(adreno_gpu)) |
1029 | clock_cntl_on = 0xaaa8aa82; | |
18397519 KD |
1030 | else if (adreno_is_a702(adreno_gpu)) |
1031 | clock_cntl_on = 0xaaaaaa82; | |
66ffb915 JM |
1032 | else |
1033 | clock_cntl_on = 0x8aa8aa82; | |
1034 | ||
af66706a | 1035 | if (adreno_is_a7xx(adreno_gpu)) { |
1f8c29e8 KD |
1036 | cgc_mode = adreno_is_a740_family(adreno_gpu) ? 0x20222 : 0x20000; |
1037 | ||
af66706a | 1038 | gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL, |
1f8c29e8 | 1039 | state ? cgc_mode : 0); |
af66706a KD |
1040 | gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, |
1041 | state ? 0x10111 : 0); | |
1042 | gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL, | |
1043 | state ? 0x5555 : 0); | |
1044 | } | |
1045 | ||
d2bcca0c NA |
1046 | if (!adreno_gpu->info->hwcg) { |
1047 | gpu_write(gpu, REG_A7XX_RBBM_CLOCK_CNTL_GLOBAL, 1); | |
1048 | gpu_write(gpu, REG_A7XX_RBBM_CGC_GLOBAL_LOAD_CMD, state ? 1 : 0); | |
1049 | ||
1050 | if (state) { | |
1051 | gpu_write(gpu, REG_A7XX_RBBM_CGC_P2S_TRIG_CMD, 1); | |
1052 | ||
1053 | if (gpu_poll_timeout(gpu, REG_A7XX_RBBM_CGC_P2S_STATUS, val, | |
1054 | val & A7XX_RBBM_CGC_P2S_STATUS_TXDONE, 1, 10)) { | |
1055 | dev_err(&gpu->pdev->dev, "RBBM_CGC_P2S_STATUS TXDONE Poll failed\n"); | |
1056 | return; | |
1057 | } | |
1058 | ||
1059 | gpu_write(gpu, REG_A7XX_RBBM_CLOCK_CNTL_GLOBAL, 0); | |
1060 | } | |
1061 | ||
1062 | return; | |
1063 | } | |
1064 | ||
4b565ca5 JC |
1065 | val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL); |
1066 | ||
1067 | /* Don't re-program the registers if they are already correct */ | |
66ffb915 | 1068 | if ((!state && !val) || (state && (val == clock_cntl_on))) |
4b565ca5 JC |
1069 | return; |
1070 | ||
1071 | /* Disable SP clock before programming HWCG registers */ | |
18397519 | 1072 | if (!adreno_is_a610_family(adreno_gpu) && !adreno_is_a7xx(adreno_gpu)) |
e7fc9398 | 1073 | gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0); |
4b565ca5 | 1074 | |
b1c53a2a JM |
1075 | for (i = 0; (reg = &adreno_gpu->info->hwcg[i], reg->offset); i++) |
1076 | gpu_write(gpu, reg->offset, state ? reg->value : 0); | |
4b565ca5 JC |
1077 | |
1078 | /* Enable SP clock */ | |
18397519 | 1079 | if (!adreno_is_a610_family(adreno_gpu) && !adreno_is_a7xx(adreno_gpu)) |
e7fc9398 | 1080 | gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1); |
4b565ca5 | 1081 | |
66ffb915 | 1082 | gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? clock_cntl_on : 0); |
4b565ca5 JC |
1083 | } |
1084 | ||
b7616b5c | 1085 | /* For a615, a616, a618, a619, a630, a640 and a680 */ |
40843403 JM |
1086 | static const u32 a6xx_protect[] = { |
1087 | A6XX_PROTECT_RDONLY(0x00000, 0x04ff), | |
1088 | A6XX_PROTECT_RDONLY(0x00501, 0x0005), | |
1089 | A6XX_PROTECT_RDONLY(0x0050b, 0x02f4), | |
1090 | A6XX_PROTECT_NORDWR(0x0050e, 0x0000), | |
1091 | A6XX_PROTECT_NORDWR(0x00510, 0x0000), | |
1092 | A6XX_PROTECT_NORDWR(0x00534, 0x0000), | |
1093 | A6XX_PROTECT_NORDWR(0x00800, 0x0082), | |
1094 | A6XX_PROTECT_NORDWR(0x008a0, 0x0008), | |
1095 | A6XX_PROTECT_NORDWR(0x008ab, 0x0024), | |
1096 | A6XX_PROTECT_RDONLY(0x008de, 0x00ae), | |
1097 | A6XX_PROTECT_NORDWR(0x00900, 0x004d), | |
1098 | A6XX_PROTECT_NORDWR(0x0098d, 0x0272), | |
1099 | A6XX_PROTECT_NORDWR(0x00e00, 0x0001), | |
1100 | A6XX_PROTECT_NORDWR(0x00e03, 0x000c), | |
1101 | A6XX_PROTECT_NORDWR(0x03c00, 0x00c3), | |
1102 | A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff), | |
1103 | A6XX_PROTECT_NORDWR(0x08630, 0x01cf), | |
1104 | A6XX_PROTECT_NORDWR(0x08e00, 0x0000), | |
1105 | A6XX_PROTECT_NORDWR(0x08e08, 0x0000), | |
1106 | A6XX_PROTECT_NORDWR(0x08e50, 0x001f), | |
1107 | A6XX_PROTECT_NORDWR(0x09624, 0x01db), | |
1108 | A6XX_PROTECT_NORDWR(0x09e70, 0x0001), | |
1109 | A6XX_PROTECT_NORDWR(0x09e78, 0x0187), | |
1110 | A6XX_PROTECT_NORDWR(0x0a630, 0x01cf), | |
1111 | A6XX_PROTECT_NORDWR(0x0ae02, 0x0000), | |
1112 | A6XX_PROTECT_NORDWR(0x0ae50, 0x032f), | |
1113 | A6XX_PROTECT_NORDWR(0x0b604, 0x0000), | |
1114 | A6XX_PROTECT_NORDWR(0x0be02, 0x0001), | |
1115 | A6XX_PROTECT_NORDWR(0x0be20, 0x17df), | |
1116 | A6XX_PROTECT_NORDWR(0x0f000, 0x0bff), | |
1117 | A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff), | |
1118 | A6XX_PROTECT_NORDWR(0x11c00, 0x0000), /* note: infinite range */ | |
1119 | }; | |
1120 | ||
1121 | /* These are for a620 and a650 */ | |
1122 | static const u32 a650_protect[] = { | |
1123 | A6XX_PROTECT_RDONLY(0x00000, 0x04ff), | |
1124 | A6XX_PROTECT_RDONLY(0x00501, 0x0005), | |
1125 | A6XX_PROTECT_RDONLY(0x0050b, 0x02f4), | |
1126 | A6XX_PROTECT_NORDWR(0x0050e, 0x0000), | |
1127 | A6XX_PROTECT_NORDWR(0x00510, 0x0000), | |
1128 | A6XX_PROTECT_NORDWR(0x00534, 0x0000), | |
1129 | A6XX_PROTECT_NORDWR(0x00800, 0x0082), | |
1130 | A6XX_PROTECT_NORDWR(0x008a0, 0x0008), | |
1131 | A6XX_PROTECT_NORDWR(0x008ab, 0x0024), | |
1132 | A6XX_PROTECT_RDONLY(0x008de, 0x00ae), | |
1133 | A6XX_PROTECT_NORDWR(0x00900, 0x004d), | |
1134 | A6XX_PROTECT_NORDWR(0x0098d, 0x0272), | |
1135 | A6XX_PROTECT_NORDWR(0x00e00, 0x0001), | |
1136 | A6XX_PROTECT_NORDWR(0x00e03, 0x000c), | |
1137 | A6XX_PROTECT_NORDWR(0x03c00, 0x00c3), | |
1138 | A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff), | |
1139 | A6XX_PROTECT_NORDWR(0x08630, 0x01cf), | |
1140 | A6XX_PROTECT_NORDWR(0x08e00, 0x0000), | |
1141 | A6XX_PROTECT_NORDWR(0x08e08, 0x0000), | |
1142 | A6XX_PROTECT_NORDWR(0x08e50, 0x001f), | |
1143 | A6XX_PROTECT_NORDWR(0x08e80, 0x027f), | |
1144 | A6XX_PROTECT_NORDWR(0x09624, 0x01db), | |
1145 | A6XX_PROTECT_NORDWR(0x09e60, 0x0011), | |
1146 | A6XX_PROTECT_NORDWR(0x09e78, 0x0187), | |
1147 | A6XX_PROTECT_NORDWR(0x0a630, 0x01cf), | |
1148 | A6XX_PROTECT_NORDWR(0x0ae02, 0x0000), | |
1149 | A6XX_PROTECT_NORDWR(0x0ae50, 0x032f), | |
1150 | A6XX_PROTECT_NORDWR(0x0b604, 0x0000), | |
1151 | A6XX_PROTECT_NORDWR(0x0b608, 0x0007), | |
1152 | A6XX_PROTECT_NORDWR(0x0be02, 0x0001), | |
1153 | A6XX_PROTECT_NORDWR(0x0be20, 0x17df), | |
1154 | A6XX_PROTECT_NORDWR(0x0f000, 0x0bff), | |
1155 | A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff), | |
1156 | A6XX_PROTECT_NORDWR(0x18400, 0x1fff), | |
1157 | A6XX_PROTECT_NORDWR(0x1a800, 0x1fff), | |
1158 | A6XX_PROTECT_NORDWR(0x1f400, 0x0443), | |
1159 | A6XX_PROTECT_RDONLY(0x1f844, 0x007b), | |
1160 | A6XX_PROTECT_NORDWR(0x1f887, 0x001b), | |
1161 | A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000), /* note: infinite range */ | |
1162 | }; | |
1163 | ||
f6d62d09 JM |
1164 | /* These are for a635 and a660 */ |
1165 | static const u32 a660_protect[] = { | |
1166 | A6XX_PROTECT_RDONLY(0x00000, 0x04ff), | |
1167 | A6XX_PROTECT_RDONLY(0x00501, 0x0005), | |
1168 | A6XX_PROTECT_RDONLY(0x0050b, 0x02f4), | |
1169 | A6XX_PROTECT_NORDWR(0x0050e, 0x0000), | |
1170 | A6XX_PROTECT_NORDWR(0x00510, 0x0000), | |
1171 | A6XX_PROTECT_NORDWR(0x00534, 0x0000), | |
1172 | A6XX_PROTECT_NORDWR(0x00800, 0x0082), | |
1173 | A6XX_PROTECT_NORDWR(0x008a0, 0x0008), | |
1174 | A6XX_PROTECT_NORDWR(0x008ab, 0x0024), | |
1175 | A6XX_PROTECT_RDONLY(0x008de, 0x00ae), | |
1176 | A6XX_PROTECT_NORDWR(0x00900, 0x004d), | |
1177 | A6XX_PROTECT_NORDWR(0x0098d, 0x0272), | |
1178 | A6XX_PROTECT_NORDWR(0x00e00, 0x0001), | |
1179 | A6XX_PROTECT_NORDWR(0x00e03, 0x000c), | |
1180 | A6XX_PROTECT_NORDWR(0x03c00, 0x00c3), | |
1181 | A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff), | |
1182 | A6XX_PROTECT_NORDWR(0x08630, 0x01cf), | |
1183 | A6XX_PROTECT_NORDWR(0x08e00, 0x0000), | |
1184 | A6XX_PROTECT_NORDWR(0x08e08, 0x0000), | |
1185 | A6XX_PROTECT_NORDWR(0x08e50, 0x001f), | |
1186 | A6XX_PROTECT_NORDWR(0x08e80, 0x027f), | |
1187 | A6XX_PROTECT_NORDWR(0x09624, 0x01db), | |
1188 | A6XX_PROTECT_NORDWR(0x09e60, 0x0011), | |
1189 | A6XX_PROTECT_NORDWR(0x09e78, 0x0187), | |
1190 | A6XX_PROTECT_NORDWR(0x0a630, 0x01cf), | |
1191 | A6XX_PROTECT_NORDWR(0x0ae02, 0x0000), | |
1192 | A6XX_PROTECT_NORDWR(0x0ae50, 0x012f), | |
1193 | A6XX_PROTECT_NORDWR(0x0b604, 0x0000), | |
1194 | A6XX_PROTECT_NORDWR(0x0b608, 0x0006), | |
1195 | A6XX_PROTECT_NORDWR(0x0be02, 0x0001), | |
1196 | A6XX_PROTECT_NORDWR(0x0be20, 0x015f), | |
1197 | A6XX_PROTECT_NORDWR(0x0d000, 0x05ff), | |
1198 | A6XX_PROTECT_NORDWR(0x0f000, 0x0bff), | |
1199 | A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff), | |
1200 | A6XX_PROTECT_NORDWR(0x18400, 0x1fff), | |
1201 | A6XX_PROTECT_NORDWR(0x1a400, 0x1fff), | |
1202 | A6XX_PROTECT_NORDWR(0x1f400, 0x0443), | |
1203 | A6XX_PROTECT_RDONLY(0x1f844, 0x007b), | |
1204 | A6XX_PROTECT_NORDWR(0x1f860, 0x0000), | |
1205 | A6XX_PROTECT_NORDWR(0x1f887, 0x001b), | |
1206 | A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000), /* note: infinite range */ | |
1207 | }; | |
1208 | ||
5e7665b5 BA |
1209 | /* These are for a690 */ |
1210 | static const u32 a690_protect[] = { | |
1211 | A6XX_PROTECT_RDONLY(0x00000, 0x004ff), | |
1212 | A6XX_PROTECT_RDONLY(0x00501, 0x00001), | |
1213 | A6XX_PROTECT_RDONLY(0x0050b, 0x002f4), | |
1214 | A6XX_PROTECT_NORDWR(0x0050e, 0x00000), | |
1215 | A6XX_PROTECT_NORDWR(0x00510, 0x00000), | |
1216 | A6XX_PROTECT_NORDWR(0x00534, 0x00000), | |
1217 | A6XX_PROTECT_NORDWR(0x00800, 0x00082), | |
1218 | A6XX_PROTECT_NORDWR(0x008a0, 0x00008), | |
1219 | A6XX_PROTECT_NORDWR(0x008ab, 0x00024), | |
bf08e979 | 1220 | A6XX_PROTECT_RDONLY(0x008de, 0x000ae), |
5e7665b5 BA |
1221 | A6XX_PROTECT_NORDWR(0x00900, 0x0004d), |
1222 | A6XX_PROTECT_NORDWR(0x0098d, 0x00272), | |
1223 | A6XX_PROTECT_NORDWR(0x00e00, 0x00001), | |
1224 | A6XX_PROTECT_NORDWR(0x00e03, 0x0000c), | |
1225 | A6XX_PROTECT_NORDWR(0x03c00, 0x000c3), | |
1226 | A6XX_PROTECT_RDONLY(0x03cc4, 0x01fff), | |
1227 | A6XX_PROTECT_NORDWR(0x08630, 0x001cf), | |
1228 | A6XX_PROTECT_NORDWR(0x08e00, 0x00000), | |
bf08e979 | 1229 | A6XX_PROTECT_NORDWR(0x08e08, 0x00007), |
5e7665b5 BA |
1230 | A6XX_PROTECT_NORDWR(0x08e50, 0x0001f), |
1231 | A6XX_PROTECT_NORDWR(0x08e80, 0x0027f), | |
1232 | A6XX_PROTECT_NORDWR(0x09624, 0x001db), | |
1233 | A6XX_PROTECT_NORDWR(0x09e60, 0x00011), | |
1234 | A6XX_PROTECT_NORDWR(0x09e78, 0x00187), | |
1235 | A6XX_PROTECT_NORDWR(0x0a630, 0x001cf), | |
1236 | A6XX_PROTECT_NORDWR(0x0ae02, 0x00000), | |
1237 | A6XX_PROTECT_NORDWR(0x0ae50, 0x0012f), | |
1238 | A6XX_PROTECT_NORDWR(0x0b604, 0x00000), | |
1239 | A6XX_PROTECT_NORDWR(0x0b608, 0x00006), | |
1240 | A6XX_PROTECT_NORDWR(0x0be02, 0x00001), | |
1241 | A6XX_PROTECT_NORDWR(0x0be20, 0x0015f), | |
1242 | A6XX_PROTECT_NORDWR(0x0d000, 0x005ff), | |
1243 | A6XX_PROTECT_NORDWR(0x0f000, 0x00bff), | |
1244 | A6XX_PROTECT_RDONLY(0x0fc00, 0x01fff), | |
1245 | A6XX_PROTECT_NORDWR(0x11c00, 0x00000), /*note: infiite range */ | |
1246 | }; | |
1247 | ||
9588d2f8 KD |
1248 | static const u32 a730_protect[] = { |
1249 | A6XX_PROTECT_RDONLY(0x00000, 0x04ff), | |
1250 | A6XX_PROTECT_RDONLY(0x0050b, 0x0058), | |
1251 | A6XX_PROTECT_NORDWR(0x0050e, 0x0000), | |
1252 | A6XX_PROTECT_NORDWR(0x00510, 0x0000), | |
1253 | A6XX_PROTECT_NORDWR(0x00534, 0x0000), | |
1254 | A6XX_PROTECT_RDONLY(0x005fb, 0x009d), | |
1255 | A6XX_PROTECT_NORDWR(0x00699, 0x01e9), | |
1256 | A6XX_PROTECT_NORDWR(0x008a0, 0x0008), | |
1257 | A6XX_PROTECT_NORDWR(0x008ab, 0x0024), | |
1258 | /* 0x008d0-0x008dd are unprotected on purpose for tools like perfetto */ | |
1259 | A6XX_PROTECT_RDONLY(0x008de, 0x0154), | |
1260 | A6XX_PROTECT_NORDWR(0x00900, 0x004d), | |
1261 | A6XX_PROTECT_NORDWR(0x0098d, 0x00b2), | |
1262 | A6XX_PROTECT_NORDWR(0x00a41, 0x01be), | |
1263 | A6XX_PROTECT_NORDWR(0x00df0, 0x0001), | |
1264 | A6XX_PROTECT_NORDWR(0x00e01, 0x0000), | |
1265 | A6XX_PROTECT_NORDWR(0x00e07, 0x0008), | |
1266 | A6XX_PROTECT_NORDWR(0x03c00, 0x00c3), | |
1267 | A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff), | |
1268 | A6XX_PROTECT_NORDWR(0x08630, 0x01cf), | |
1269 | A6XX_PROTECT_NORDWR(0x08e00, 0x0000), | |
1270 | A6XX_PROTECT_NORDWR(0x08e08, 0x0000), | |
1271 | A6XX_PROTECT_NORDWR(0x08e50, 0x001f), | |
1272 | A6XX_PROTECT_NORDWR(0x08e80, 0x0280), | |
1273 | A6XX_PROTECT_NORDWR(0x09624, 0x01db), | |
1274 | A6XX_PROTECT_NORDWR(0x09e40, 0x0000), | |
1275 | A6XX_PROTECT_NORDWR(0x09e64, 0x000d), | |
1276 | A6XX_PROTECT_NORDWR(0x09e78, 0x0187), | |
1277 | A6XX_PROTECT_NORDWR(0x0a630, 0x01cf), | |
1278 | A6XX_PROTECT_NORDWR(0x0ae02, 0x0000), | |
1279 | A6XX_PROTECT_NORDWR(0x0ae50, 0x000f), | |
1280 | A6XX_PROTECT_NORDWR(0x0ae66, 0x0003), | |
1281 | A6XX_PROTECT_NORDWR(0x0ae6f, 0x0003), | |
1282 | A6XX_PROTECT_NORDWR(0x0b604, 0x0003), | |
1283 | A6XX_PROTECT_NORDWR(0x0ec00, 0x0fff), | |
1284 | A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff), | |
1285 | A6XX_PROTECT_NORDWR(0x18400, 0x0053), | |
1286 | A6XX_PROTECT_RDONLY(0x18454, 0x0004), | |
1287 | A6XX_PROTECT_NORDWR(0x18459, 0x1fff), | |
1288 | A6XX_PROTECT_NORDWR(0x1a459, 0x1fff), | |
1289 | A6XX_PROTECT_NORDWR(0x1c459, 0x1fff), | |
1290 | A6XX_PROTECT_NORDWR(0x1f400, 0x0443), | |
1291 | A6XX_PROTECT_RDONLY(0x1f844, 0x007b), | |
1292 | A6XX_PROTECT_NORDWR(0x1f860, 0x0000), | |
1293 | A6XX_PROTECT_NORDWR(0x1f878, 0x002a), | |
1294 | /* CP_PROTECT_REG[44, 46] are left untouched! */ | |
1295 | 0, | |
1296 | 0, | |
1297 | 0, | |
1298 | A6XX_PROTECT_NORDWR(0x1f8c0, 0x00000), | |
1299 | }; | |
1300 | ||
40843403 JM |
1301 | static void a6xx_set_cp_protect(struct msm_gpu *gpu) |
1302 | { | |
1303 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
1304 | const u32 *regs = a6xx_protect; | |
cca96584 | 1305 | unsigned i, count, count_max; |
40843403 | 1306 | |
18397519 | 1307 | if (adreno_is_a650(adreno_gpu) || adreno_is_a702(adreno_gpu)) { |
40843403 JM |
1308 | regs = a650_protect; |
1309 | count = ARRAY_SIZE(a650_protect); | |
1310 | count_max = 48; | |
cca96584 | 1311 | BUILD_BUG_ON(ARRAY_SIZE(a650_protect) > 48); |
5e7665b5 BA |
1312 | } else if (adreno_is_a690(adreno_gpu)) { |
1313 | regs = a690_protect; | |
1314 | count = ARRAY_SIZE(a690_protect); | |
1315 | count_max = 48; | |
1316 | BUILD_BUG_ON(ARRAY_SIZE(a690_protect) > 48); | |
192f4ee3 | 1317 | } else if (adreno_is_a660_family(adreno_gpu)) { |
f6d62d09 JM |
1318 | regs = a660_protect; |
1319 | count = ARRAY_SIZE(a660_protect); | |
1320 | count_max = 48; | |
cca96584 | 1321 | BUILD_BUG_ON(ARRAY_SIZE(a660_protect) > 48); |
d2bcca0c NA |
1322 | } else if (adreno_is_a730(adreno_gpu) || |
1323 | adreno_is_a740(adreno_gpu) || | |
1324 | adreno_is_a750(adreno_gpu)) { | |
9588d2f8 KD |
1325 | regs = a730_protect; |
1326 | count = ARRAY_SIZE(a730_protect); | |
1327 | count_max = 48; | |
1328 | BUILD_BUG_ON(ARRAY_SIZE(a730_protect) > 48); | |
cca96584 RC |
1329 | } else { |
1330 | regs = a6xx_protect; | |
1331 | count = ARRAY_SIZE(a6xx_protect); | |
1332 | count_max = 32; | |
1333 | BUILD_BUG_ON(ARRAY_SIZE(a6xx_protect) > 32); | |
40843403 JM |
1334 | } |
1335 | ||
1336 | /* | |
1337 | * Enable access protection to privileged registers, fault on an access | |
1338 | * protect violation and select the last span to protect from the start | |
1339 | * address all the way to the end of the register address space | |
1340 | */ | |
02a726fc KD |
1341 | gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, |
1342 | A6XX_CP_PROTECT_CNTL_ACCESS_PROT_EN | | |
1343 | A6XX_CP_PROTECT_CNTL_ACCESS_FAULT_ON_VIOL_EN | | |
1344 | A6XX_CP_PROTECT_CNTL_LAST_SPAN_INF_RANGE); | |
40843403 | 1345 | |
29af7605 KD |
1346 | for (i = 0; i < count - 1; i++) { |
1347 | /* Intentionally skip writing to some registers */ | |
1348 | if (regs[i]) | |
1349 | gpu_write(gpu, REG_A6XX_CP_PROTECT(i), regs[i]); | |
1350 | } | |
40843403 JM |
1351 | /* last CP_PROTECT to have "infinite" length on the last entry */ |
1352 | gpu_write(gpu, REG_A6XX_CP_PROTECT(count_max - 1), regs[i]); | |
1353 | } | |
1354 | ||
8814455a | 1355 | static void a6xx_calc_ubwc_config(struct adreno_gpu *gpu) |
d0bac4e9 | 1356 | { |
df5bb404 | 1357 | /* Unknown, introduced with A650 family, related to UBWC mode/ver 4 */ |
8814455a | 1358 | gpu->ubwc_config.rgb565_predicator = 0; |
df5bb404 | 1359 | /* Unknown, introduced with A650 family */ |
8814455a | 1360 | gpu->ubwc_config.uavflagprd_inv = 0; |
df5bb404 | 1361 | /* Whether the minimum access length is 64 bits */ |
8814455a | 1362 | gpu->ubwc_config.min_acc_len = 0; |
df5bb404 | 1363 | /* Entirely magic, per-GPU-gen value */ |
8814455a | 1364 | gpu->ubwc_config.ubwc_mode = 0; |
df5bb404 KD |
1365 | /* |
1366 | * The Highest Bank Bit value represents the bit of the highest DDR bank. | |
8814455a | 1367 | * This should ideally use DRAM type detection. |
df5bb404 | 1368 | */ |
8814455a | 1369 | gpu->ubwc_config.highest_bank_bit = 15; |
d0bac4e9 | 1370 | |
8814455a | 1371 | if (adreno_is_a610(gpu)) { |
6a0dbcd2 | 1372 | gpu->ubwc_config.highest_bank_bit = 13; |
8814455a CA |
1373 | gpu->ubwc_config.min_acc_len = 1; |
1374 | gpu->ubwc_config.ubwc_mode = 1; | |
e7fc9398 KD |
1375 | } |
1376 | ||
8814455a | 1377 | if (adreno_is_a618(gpu)) |
0d7dfc79 | 1378 | gpu->ubwc_config.highest_bank_bit = 14; |
d0bac4e9 | 1379 | |
9dc23cba LW |
1380 | if (adreno_is_a619(gpu)) |
1381 | /* TODO: Should be 14 but causes corruption at e.g. 1920x1200 on DP */ | |
1382 | gpu->ubwc_config.highest_bank_bit = 13; | |
1383 | ||
8814455a CA |
1384 | if (adreno_is_a619_holi(gpu)) |
1385 | gpu->ubwc_config.highest_bank_bit = 13; | |
8296ff0a | 1386 | |
8814455a CA |
1387 | if (adreno_is_a640_family(gpu)) |
1388 | gpu->ubwc_config.amsbc = 1; | |
d0bac4e9 | 1389 | |
8814455a CA |
1390 | if (adreno_is_a650(gpu) || |
1391 | adreno_is_a660(gpu) || | |
1392 | adreno_is_a690(gpu) || | |
1393 | adreno_is_a730(gpu) || | |
1394 | adreno_is_a740_family(gpu)) { | |
d0bac4e9 | 1395 | /* TODO: get ddr type from bootloader and use 2 for LPDDR4 */ |
8814455a CA |
1396 | gpu->ubwc_config.highest_bank_bit = 16; |
1397 | gpu->ubwc_config.amsbc = 1; | |
1398 | gpu->ubwc_config.rgb565_predicator = 1; | |
1399 | gpu->ubwc_config.uavflagprd_inv = 2; | |
d0bac4e9 JM |
1400 | } |
1401 | ||
8814455a CA |
1402 | if (adreno_is_7c3(gpu)) { |
1403 | gpu->ubwc_config.highest_bank_bit = 14; | |
1404 | gpu->ubwc_config.amsbc = 1; | |
1405 | gpu->ubwc_config.rgb565_predicator = 1; | |
1406 | gpu->ubwc_config.uavflagprd_inv = 2; | |
192f4ee3 | 1407 | } |
18397519 KD |
1408 | |
1409 | if (adreno_is_a702(gpu)) { | |
1410 | gpu->ubwc_config.highest_bank_bit = 14; | |
1411 | gpu->ubwc_config.min_acc_len = 1; | |
1412 | gpu->ubwc_config.ubwc_mode = 2; | |
1413 | } | |
8814455a CA |
1414 | } |
1415 | ||
1416 | static void a6xx_set_ubwc_config(struct msm_gpu *gpu) | |
1417 | { | |
1418 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
1419 | /* | |
1420 | * We subtract 13 from the highest bank bit (13 is the minimum value | |
1421 | * allowed by hw) and write the lowest two bits of the remaining value | |
1422 | * as hbb_lo and the one above it as hbb_hi to the hardware. | |
1423 | */ | |
1424 | BUG_ON(adreno_gpu->ubwc_config.highest_bank_bit < 13); | |
1425 | u32 hbb = adreno_gpu->ubwc_config.highest_bank_bit - 13; | |
1426 | u32 hbb_hi = hbb >> 2; | |
1427 | u32 hbb_lo = hbb & 3; | |
192f4ee3 | 1428 | |
d0bac4e9 | 1429 | gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL, |
8814455a CA |
1430 | adreno_gpu->ubwc_config.rgb565_predicator << 11 | |
1431 | hbb_hi << 10 | adreno_gpu->ubwc_config.amsbc << 4 | | |
1432 | adreno_gpu->ubwc_config.min_acc_len << 3 | | |
1433 | hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode); | |
df5bb404 KD |
1434 | |
1435 | gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, hbb_hi << 4 | | |
8814455a CA |
1436 | adreno_gpu->ubwc_config.min_acc_len << 3 | |
1437 | hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode); | |
df5bb404 KD |
1438 | |
1439 | gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, hbb_hi << 10 | | |
8814455a CA |
1440 | adreno_gpu->ubwc_config.uavflagprd_inv << 4 | |
1441 | adreno_gpu->ubwc_config.min_acc_len << 3 | | |
1442 | hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode); | |
df5bb404 | 1443 | |
af66706a KD |
1444 | if (adreno_is_a7xx(adreno_gpu)) |
1445 | gpu_write(gpu, REG_A7XX_GRAS_NC_MODE_CNTL, | |
1446 | FIELD_PREP(GENMASK(8, 5), hbb_lo)); | |
1447 | ||
8814455a CA |
1448 | gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL, |
1449 | adreno_gpu->ubwc_config.min_acc_len << 23 | hbb_lo << 21); | |
d0bac4e9 JM |
1450 | } |
1451 | ||
4b565ca5 JC |
1452 | static int a6xx_cp_init(struct msm_gpu *gpu) |
1453 | { | |
1454 | struct msm_ringbuffer *ring = gpu->rb[0]; | |
1455 | ||
1456 | OUT_PKT7(ring, CP_ME_INIT, 8); | |
1457 | ||
1458 | OUT_RING(ring, 0x0000002f); | |
1459 | ||
1460 | /* Enable multiple hardware contexts */ | |
1461 | OUT_RING(ring, 0x00000003); | |
1462 | ||
1463 | /* Enable error detection */ | |
1464 | OUT_RING(ring, 0x20000000); | |
1465 | ||
1466 | /* Don't enable header dump */ | |
1467 | OUT_RING(ring, 0x00000000); | |
1468 | OUT_RING(ring, 0x00000000); | |
1469 | ||
1470 | /* No workarounds enabled */ | |
1471 | OUT_RING(ring, 0x00000000); | |
1472 | ||
1473 | /* Pad rest of the cmds with 0's */ | |
1474 | OUT_RING(ring, 0x00000000); | |
1475 | OUT_RING(ring, 0x00000000); | |
1476 | ||
1477 | a6xx_flush(gpu, ring); | |
1478 | return a6xx_idle(gpu, ring) ? 0 : -EINVAL; | |
1479 | } | |
1480 | ||
af66706a KD |
1481 | static int a7xx_cp_init(struct msm_gpu *gpu) |
1482 | { | |
1483 | struct msm_ringbuffer *ring = gpu->rb[0]; | |
1484 | u32 mask; | |
1485 | ||
1486 | /* Disable concurrent binning before sending CP init */ | |
1487 | OUT_PKT7(ring, CP_THREAD_CONTROL, 1); | |
1488 | OUT_RING(ring, BIT(27)); | |
1489 | ||
1490 | OUT_PKT7(ring, CP_ME_INIT, 7); | |
1491 | ||
1492 | /* Use multiple HW contexts */ | |
1493 | mask = BIT(0); | |
1494 | ||
1495 | /* Enable error detection */ | |
1496 | mask |= BIT(1); | |
1497 | ||
1498 | /* Set default reset state */ | |
1499 | mask |= BIT(3); | |
1500 | ||
1501 | /* Disable save/restore of performance counters across preemption */ | |
1502 | mask |= BIT(6); | |
1503 | ||
1504 | /* Enable the register init list with the spinlock */ | |
1505 | mask |= BIT(8); | |
1506 | ||
1507 | OUT_RING(ring, mask); | |
1508 | ||
1509 | /* Enable multiple hardware contexts */ | |
1510 | OUT_RING(ring, 0x00000003); | |
1511 | ||
1512 | /* Enable error detection */ | |
1513 | OUT_RING(ring, 0x20000000); | |
1514 | ||
1515 | /* Operation mode mask */ | |
1516 | OUT_RING(ring, 0x00000002); | |
1517 | ||
1518 | /* *Don't* send a power up reg list for concurrent binning (TODO) */ | |
1519 | /* Lo address */ | |
1520 | OUT_RING(ring, 0x00000000); | |
1521 | /* Hi address */ | |
1522 | OUT_RING(ring, 0x00000000); | |
1523 | /* BIT(31) set => read the regs from the list */ | |
1524 | OUT_RING(ring, 0x00000000); | |
1525 | ||
1526 | a6xx_flush(gpu, ring); | |
1527 | return a6xx_idle(gpu, ring) ? 0 : -EINVAL; | |
1528 | } | |
1529 | ||
8490f02a JC |
1530 | /* |
1531 | * Check that the microcode version is new enough to include several key | |
1532 | * security fixes. Return true if the ucode is safe. | |
1533 | */ | |
1534 | static bool a6xx_ucode_check_version(struct a6xx_gpu *a6xx_gpu, | |
d3a569fc JC |
1535 | struct drm_gem_object *obj) |
1536 | { | |
8490f02a JC |
1537 | struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; |
1538 | struct msm_gpu *gpu = &adreno_gpu->base; | |
f3a6b02c | 1539 | const char *sqe_name = adreno_gpu->info->fw[ADRENO_FW_SQE]; |
96c876f1 | 1540 | u32 *buf = msm_gem_get_vaddr(obj); |
8490f02a | 1541 | bool ret = false; |
d3a569fc JC |
1542 | |
1543 | if (IS_ERR(buf)) | |
8490f02a | 1544 | return false; |
d3a569fc | 1545 | |
af66706a | 1546 | /* A7xx is safe! */ |
18397519 | 1547 | if (adreno_is_a7xx(adreno_gpu) || adreno_is_a702(adreno_gpu)) |
af66706a KD |
1548 | return true; |
1549 | ||
d3a569fc | 1550 | /* |
8490f02a JC |
1551 | * Targets up to a640 (a618, a630 and a640) need to check for a |
1552 | * microcode version that is patched to support the whereami opcode or | |
1553 | * one that is new enough to include it by default. | |
f6d62d09 JM |
1554 | * |
1555 | * a650 tier targets don't need whereami but still need to be | |
1556 | * equal to or newer than 0.95 for other security fixes | |
1557 | * | |
1558 | * a660 targets have all the critical security fixes from the start | |
d3a569fc | 1559 | */ |
f3a6b02c | 1560 | if (!strcmp(sqe_name, "a630_sqe.fw")) { |
8490f02a JC |
1561 | /* |
1562 | * If the lowest nibble is 0xa that is an indication that this | |
1563 | * microcode has been patched. The actual version is in dword | |
1564 | * [3] but we only care about the patchlevel which is the lowest | |
1565 | * nibble of dword [3] | |
1566 | * | |
1567 | * Otherwise check that the firmware is greater than or equal | |
1568 | * to 1.90 which was the first version that had this fix built | |
1569 | * in | |
1570 | */ | |
1571 | if ((((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1) || | |
1572 | (buf[0] & 0xfff) >= 0x190) { | |
1573 | a6xx_gpu->has_whereami = true; | |
1574 | ret = true; | |
1575 | goto out; | |
1576 | } | |
1577 | ||
1578 | DRM_DEV_ERROR(&gpu->pdev->dev, | |
1579 | "a630 SQE ucode is too old. Have version %x need at least %x\n", | |
1580 | buf[0] & 0xfff, 0x190); | |
f3a6b02c | 1581 | } else if (!strcmp(sqe_name, "a650_sqe.fw")) { |
f6d62d09 JM |
1582 | if ((buf[0] & 0xfff) >= 0x095) { |
1583 | ret = true; | |
1584 | goto out; | |
8490f02a | 1585 | } |
d3a569fc | 1586 | |
f6d62d09 JM |
1587 | DRM_DEV_ERROR(&gpu->pdev->dev, |
1588 | "a650 SQE ucode is too old. Have version %x need at least %x\n", | |
1589 | buf[0] & 0xfff, 0x095); | |
f3a6b02c | 1590 | } else if (!strcmp(sqe_name, "a660_sqe.fw")) { |
f6d62d09 JM |
1591 | ret = true; |
1592 | } else { | |
1593 | DRM_DEV_ERROR(&gpu->pdev->dev, | |
1594 | "unknown GPU, add it to a6xx_ucode_check_version()!!\n"); | |
8490f02a JC |
1595 | } |
1596 | out: | |
d3a569fc | 1597 | msm_gem_put_vaddr(obj); |
8490f02a | 1598 | return ret; |
d3a569fc JC |
1599 | } |
1600 | ||
8ead9678 | 1601 | static int a6xx_ucode_load(struct msm_gpu *gpu) |
4b565ca5 JC |
1602 | { |
1603 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
1604 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
1605 | ||
1606 | if (!a6xx_gpu->sqe_bo) { | |
1607 | a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu, | |
1608 | adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova); | |
1609 | ||
1610 | if (IS_ERR(a6xx_gpu->sqe_bo)) { | |
1611 | int ret = PTR_ERR(a6xx_gpu->sqe_bo); | |
1612 | ||
1613 | a6xx_gpu->sqe_bo = NULL; | |
1614 | DRM_DEV_ERROR(&gpu->pdev->dev, | |
1615 | "Could not allocate SQE ucode: %d\n", ret); | |
1616 | ||
1617 | return ret; | |
1618 | } | |
0815d774 JC |
1619 | |
1620 | msm_gem_object_set_name(a6xx_gpu->sqe_bo, "sqefw"); | |
8490f02a JC |
1621 | if (!a6xx_ucode_check_version(a6xx_gpu, a6xx_gpu->sqe_bo)) { |
1622 | msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace); | |
1623 | drm_gem_object_put(a6xx_gpu->sqe_bo); | |
1624 | ||
1625 | a6xx_gpu->sqe_bo = NULL; | |
1626 | return -EPERM; | |
1627 | } | |
4b565ca5 JC |
1628 | } |
1629 | ||
8ead9678 RC |
1630 | /* |
1631 | * Expanded APRIV and targets that support WHERE_AM_I both need a | |
1632 | * privileged buffer to store the RPTR shadow | |
1633 | */ | |
1634 | if ((adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) && | |
1635 | !a6xx_gpu->shadow_bo) { | |
1636 | a6xx_gpu->shadow = msm_gem_kernel_new(gpu->dev, | |
1637 | sizeof(u32) * gpu->nr_rings, | |
1638 | MSM_BO_WC | MSM_BO_MAP_PRIV, | |
1639 | gpu->aspace, &a6xx_gpu->shadow_bo, | |
1640 | &a6xx_gpu->shadow_iova); | |
1641 | ||
1642 | if (IS_ERR(a6xx_gpu->shadow)) | |
1643 | return PTR_ERR(a6xx_gpu->shadow); | |
1644 | ||
1645 | msm_gem_object_set_name(a6xx_gpu->shadow_bo, "shadow"); | |
1646 | } | |
4b565ca5 JC |
1647 | |
1648 | return 0; | |
1649 | } | |
1650 | ||
abccb9fe JC |
1651 | static int a6xx_zap_shader_init(struct msm_gpu *gpu) |
1652 | { | |
1653 | static bool loaded; | |
1654 | int ret; | |
1655 | ||
1656 | if (loaded) | |
1657 | return 0; | |
1658 | ||
1659 | ret = adreno_zap_shader_load(gpu, GPU_PAS_ID); | |
1660 | ||
1661 | loaded = !ret; | |
1662 | return ret; | |
1663 | } | |
1664 | ||
4b565ca5 | 1665 | #define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \ |
af66706a KD |
1666 | A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \ |
1667 | A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \ | |
1668 | A6XX_RBBM_INT_0_MASK_CP_IB2 | \ | |
1669 | A6XX_RBBM_INT_0_MASK_CP_IB1 | \ | |
1670 | A6XX_RBBM_INT_0_MASK_CP_RB | \ | |
1671 | A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \ | |
1672 | A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \ | |
1673 | A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \ | |
1674 | A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \ | |
1675 | A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR) | |
1676 | ||
1677 | #define A7XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \ | |
1678 | A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \ | |
1679 | A6XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR | \ | |
1680 | A6XX_RBBM_INT_0_MASK_CP_SW | \ | |
1681 | A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \ | |
1682 | A6XX_RBBM_INT_0_MASK_PM4CPINTERRUPT | \ | |
1683 | A6XX_RBBM_INT_0_MASK_CP_RB_DONE_TS | \ | |
1684 | A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \ | |
1685 | A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \ | |
1686 | A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \ | |
1687 | A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \ | |
1688 | A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR | \ | |
1689 | A6XX_RBBM_INT_0_MASK_TSBWRITEERROR) | |
1690 | ||
1691 | #define A7XX_APRIV_MASK (A6XX_CP_APRIV_CNTL_ICACHE | \ | |
1692 | A6XX_CP_APRIV_CNTL_RBFETCH | \ | |
1693 | A6XX_CP_APRIV_CNTL_RBPRIVLEVEL | \ | |
1694 | A6XX_CP_APRIV_CNTL_RBRPWB) | |
1695 | ||
1696 | #define A7XX_BR_APRIVMASK (A7XX_APRIV_MASK | \ | |
1697 | A6XX_CP_APRIV_CNTL_CDREAD | \ | |
1698 | A6XX_CP_APRIV_CNTL_CDWRITE) | |
4b565ca5 | 1699 | |
f6f59072 | 1700 | static int hw_init(struct msm_gpu *gpu) |
4b565ca5 JC |
1701 | { |
1702 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
1703 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
5a903a44 | 1704 | struct a6xx_gmu *gmu = &a6xx_gpu->gmu; |
1f8c29e8 | 1705 | u64 gmem_range_min; |
4b565ca5 JC |
1706 | int ret; |
1707 | ||
5a903a44 KD |
1708 | if (!adreno_has_gmu_wrapper(adreno_gpu)) { |
1709 | /* Make sure the GMU keeps the GPU on while we set it up */ | |
34b149ec KD |
1710 | ret = a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); |
1711 | if (ret) | |
1712 | return ret; | |
5a903a44 | 1713 | } |
4b565ca5 | 1714 | |
3a9dd708 | 1715 | /* Clear GBIF halt in case GX domain was not collapsed */ |
8296ff0a KD |
1716 | if (adreno_is_a619_holi(adreno_gpu)) { |
1717 | gpu_write(gpu, REG_A6XX_GBIF_HALT, 0); | |
1718 | gpu_write(gpu, REG_A6XX_RBBM_GPR0_CNTL, 0); | |
1719 | /* Let's make extra sure that the GPU can access the memory.. */ | |
1720 | mb(); | |
1721 | } else if (a6xx_has_gbif(adreno_gpu)) { | |
05a23a76 | 1722 | gpu_write(gpu, REG_A6XX_GBIF_HALT, 0); |
3a9dd708 | 1723 | gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 0); |
05a23a76 KD |
1724 | /* Let's make extra sure that the GPU can access the memory.. */ |
1725 | mb(); | |
1726 | } | |
3a9dd708 | 1727 | |
b7753280 KD |
1728 | /* Some GPUs are stubborn and take their sweet time to unhalt GBIF! */ |
1729 | if (adreno_is_a7xx(adreno_gpu) && a6xx_has_gbif(adreno_gpu)) | |
1730 | spin_until(!gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK)); | |
1731 | ||
4b565ca5 JC |
1732 | gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0); |
1733 | ||
8296ff0a KD |
1734 | if (adreno_is_a619_holi(adreno_gpu)) |
1735 | a6xx_sptprac_enable(gmu); | |
1736 | ||
4b565ca5 JC |
1737 | /* |
1738 | * Disable the trusted memory range - we don't actually supported secure | |
1739 | * memory rendering at this point in time and we don't want to block off | |
1740 | * part of the virtual memory space. | |
1741 | */ | |
f73343fa | 1742 | gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE, 0x00000000); |
4b565ca5 JC |
1743 | gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000); |
1744 | ||
af66706a KD |
1745 | if (!adreno_is_a7xx(adreno_gpu)) { |
1746 | /* Turn on 64 bit addressing for all blocks */ | |
1747 | gpu_write(gpu, REG_A6XX_CP_ADDR_MODE_CNTL, 0x1); | |
1748 | gpu_write(gpu, REG_A6XX_VSC_ADDR_MODE_CNTL, 0x1); | |
1749 | gpu_write(gpu, REG_A6XX_GRAS_ADDR_MODE_CNTL, 0x1); | |
1750 | gpu_write(gpu, REG_A6XX_RB_ADDR_MODE_CNTL, 0x1); | |
1751 | gpu_write(gpu, REG_A6XX_PC_ADDR_MODE_CNTL, 0x1); | |
1752 | gpu_write(gpu, REG_A6XX_HLSQ_ADDR_MODE_CNTL, 0x1); | |
1753 | gpu_write(gpu, REG_A6XX_VFD_ADDR_MODE_CNTL, 0x1); | |
1754 | gpu_write(gpu, REG_A6XX_VPC_ADDR_MODE_CNTL, 0x1); | |
1755 | gpu_write(gpu, REG_A6XX_UCHE_ADDR_MODE_CNTL, 0x1); | |
1756 | gpu_write(gpu, REG_A6XX_SP_ADDR_MODE_CNTL, 0x1); | |
1757 | gpu_write(gpu, REG_A6XX_TPL1_ADDR_MODE_CNTL, 0x1); | |
1758 | gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1); | |
1759 | } | |
adf151c2 | 1760 | |
b1c53a2a JM |
1761 | /* enable hardware clockgating */ |
1762 | a6xx_set_hwcg(gpu, true); | |
4b565ca5 | 1763 | |
e812744c | 1764 | /* VBIF/GBIF start*/ |
18397519 | 1765 | if (adreno_is_a610_family(adreno_gpu) || |
e7fc9398 | 1766 | adreno_is_a640_family(adreno_gpu) || |
af66706a KD |
1767 | adreno_is_a650_family(adreno_gpu) || |
1768 | adreno_is_a7xx(adreno_gpu)) { | |
24e6938e JM |
1769 | gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE0, 0x00071620); |
1770 | gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE1, 0x00071620); | |
1771 | gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE2, 0x00071620); | |
1772 | gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620); | |
af66706a KD |
1773 | gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, |
1774 | adreno_is_a7xx(adreno_gpu) ? 0x2120212 : 0x3); | |
24e6938e JM |
1775 | } else { |
1776 | gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3); | |
1777 | } | |
1778 | ||
e812744c SM |
1779 | if (adreno_is_a630(adreno_gpu)) |
1780 | gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009); | |
4b565ca5 | 1781 | |
af66706a KD |
1782 | if (adreno_is_a7xx(adreno_gpu)) |
1783 | gpu_write(gpu, REG_A6XX_UCHE_GBIF_GX_CONFIG, 0x10240e0); | |
1784 | ||
4b565ca5 JC |
1785 | /* Make all blocks contribute to the GPU BUSY perf counter */ |
1786 | gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff); | |
1787 | ||
1788 | /* Disable L2 bypass in the UCHE */ | |
af66706a KD |
1789 | if (adreno_is_a7xx(adreno_gpu)) { |
1790 | gpu_write64(gpu, REG_A6XX_UCHE_TRAP_BASE, 0x0001fffffffff000llu); | |
1791 | gpu_write64(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE, 0x0001fffffffff000llu); | |
1792 | } else { | |
1793 | gpu_write64(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX, 0x0001ffffffffffc0llu); | |
1794 | gpu_write64(gpu, REG_A6XX_UCHE_TRAP_BASE, 0x0001fffffffff000llu); | |
1795 | gpu_write64(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE, 0x0001fffffffff000llu); | |
1796 | } | |
4b565ca5 | 1797 | |
9588d2f8 | 1798 | if (!(adreno_is_a650_family(adreno_gpu) || |
18397519 | 1799 | adreno_is_a702(adreno_gpu) || |
9588d2f8 | 1800 | adreno_is_a730(adreno_gpu))) { |
1f8c29e8 KD |
1801 | gmem_range_min = adreno_is_a740_family(adreno_gpu) ? SZ_16M : SZ_1M; |
1802 | ||
24e6938e | 1803 | /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */ |
1f8c29e8 | 1804 | gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN, gmem_range_min); |
4b565ca5 | 1805 | |
f73343fa | 1806 | gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX, |
1f8c29e8 | 1807 | gmem_range_min + adreno_gpu->info->gmem - 1); |
24e6938e | 1808 | } |
4b565ca5 | 1809 | |
af66706a KD |
1810 | if (adreno_is_a7xx(adreno_gpu)) |
1811 | gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, BIT(23)); | |
1812 | else { | |
1813 | gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804); | |
1814 | gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4); | |
1815 | } | |
4b565ca5 | 1816 | |
e7fc9398 | 1817 | if (adreno_is_a640_family(adreno_gpu) || adreno_is_a650_family(adreno_gpu)) { |
24e6938e | 1818 | gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140); |
e7fc9398 | 1819 | gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c); |
18397519 | 1820 | } else if (adreno_is_a610_family(adreno_gpu)) { |
e7fc9398 KD |
1821 | gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x00800060); |
1822 | gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x40201b16); | |
af66706a | 1823 | } else if (!adreno_is_a7xx(adreno_gpu)) { |
24e6938e | 1824 | gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0); |
e7fc9398 KD |
1825 | gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c); |
1826 | } | |
4b565ca5 | 1827 | |
192f4ee3 | 1828 | if (adreno_is_a660_family(adreno_gpu)) |
f6d62d09 JM |
1829 | gpu_write(gpu, REG_A6XX_CP_LPAC_PROG_FIFO_SIZE, 0x00000020); |
1830 | ||
4b565ca5 | 1831 | /* Setting the mem pool size */ |
e7fc9398 KD |
1832 | if (adreno_is_a610(adreno_gpu)) { |
1833 | gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 48); | |
1834 | gpu_write(gpu, REG_A6XX_CP_MEM_POOL_DBG_ADDR, 47); | |
18397519 KD |
1835 | } else if (adreno_is_a702(adreno_gpu)) { |
1836 | gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 64); | |
1837 | gpu_write(gpu, REG_A6XX_CP_MEM_POOL_DBG_ADDR, 63); | |
af66706a | 1838 | } else if (!adreno_is_a7xx(adreno_gpu)) |
e7fc9398 | 1839 | gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128); |
4b565ca5 | 1840 | |
564499f5 JM |
1841 | /* Setting the primFifo thresholds default values, |
1842 | * and vccCacheSkipDis=1 bit (0x200) for A640 and newer | |
1843 | */ | |
18397519 KD |
1844 | if (adreno_is_a702(adreno_gpu)) |
1845 | gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x0000c000); | |
1846 | else if (adreno_is_a690(adreno_gpu)) | |
07e6de73 DP |
1847 | gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00800200); |
1848 | else if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu)) | |
840d10b6 | 1849 | gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200); |
083cc3a4 | 1850 | else if (adreno_is_a640_family(adreno_gpu) || adreno_is_7c3(adreno_gpu)) |
564499f5 | 1851 | gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00200200); |
192f4ee3 AO |
1852 | else if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu)) |
1853 | gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200); | |
3e90044d KD |
1854 | else if (adreno_is_a619(adreno_gpu)) |
1855 | gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00018000); | |
e7fc9398 KD |
1856 | else if (adreno_is_a610(adreno_gpu)) |
1857 | gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00080000); | |
af66706a | 1858 | else if (!adreno_is_a7xx(adreno_gpu)) |
564499f5 | 1859 | gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00180000); |
4b565ca5 JC |
1860 | |
1861 | /* Set the AHB default slave response to "ERROR" */ | |
1862 | gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1); | |
1863 | ||
1864 | /* Turn on performance counters */ | |
1865 | gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1); | |
1866 | ||
af66706a KD |
1867 | if (adreno_is_a7xx(adreno_gpu)) { |
1868 | /* Turn on the IFPC counter (countable 4 on XOCLK4) */ | |
1869 | gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1, | |
1870 | FIELD_PREP(GENMASK(7, 0), 0x4)); | |
1871 | } | |
1872 | ||
4b565ca5 | 1873 | /* Select CP0 to always count cycles */ |
cc4c26d4 | 1874 | gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL(0), PERF_CP_ALWAYS_COUNT); |
4b565ca5 | 1875 | |
d0bac4e9 | 1876 | a6xx_set_ubwc_config(gpu); |
4b565ca5 JC |
1877 | |
1878 | /* Enable fault detection */ | |
1f8c29e8 KD |
1879 | if (adreno_is_a730(adreno_gpu) || |
1880 | adreno_is_a740_family(adreno_gpu)) | |
9588d2f8 | 1881 | gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0xcfffff); |
07e6de73 DP |
1882 | else if (adreno_is_a690(adreno_gpu)) |
1883 | gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x4fffff); | |
9588d2f8 | 1884 | else if (adreno_is_a619(adreno_gpu)) |
3e90044d | 1885 | gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x3fffff); |
18397519 | 1886 | else if (adreno_is_a610(adreno_gpu) || adreno_is_a702(adreno_gpu)) |
e7fc9398 KD |
1887 | gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x3ffff); |
1888 | else | |
1889 | gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x1fffff); | |
4b565ca5 | 1890 | |
cf1aaa7d | 1891 | gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, BIT(7) | 0x1); |
4b565ca5 | 1892 | |
24e6938e | 1893 | /* Set weights for bicubic filtering */ |
f6d62d09 | 1894 | if (adreno_is_a650_family(adreno_gpu)) { |
24e6938e JM |
1895 | gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0, 0); |
1896 | gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1, | |
1897 | 0x3fe05ff4); | |
1898 | gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2, | |
1899 | 0x3fa0ebee); | |
1900 | gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3, | |
1901 | 0x3f5193ed); | |
1902 | gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4, | |
1903 | 0x3f0243f0); | |
1904 | } | |
1905 | ||
30f55f3f KD |
1906 | /* Set up the CX GMU counter 0 to count busy ticks */ |
1907 | gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, 0xff000000); | |
1908 | ||
1909 | /* Enable the power counter */ | |
1910 | gmu_rmw(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, 0xff, BIT(5)); | |
1911 | gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1); | |
1912 | ||
4b565ca5 | 1913 | /* Protect registers from the CP */ |
40843403 | 1914 | a6xx_set_cp_protect(gpu); |
4b565ca5 | 1915 | |
192f4ee3 | 1916 | if (adreno_is_a660_family(adreno_gpu)) { |
07e6de73 DP |
1917 | if (adreno_is_a690(adreno_gpu)) |
1918 | gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, 0x00028801); | |
1919 | else | |
1920 | gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, 0x1); | |
f6d62d09 | 1921 | gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x0); |
18397519 KD |
1922 | } else if (adreno_is_a702(adreno_gpu)) { |
1923 | /* Something to do with the HLSQ cluster */ | |
1924 | gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, BIT(24)); | |
f6d62d09 JM |
1925 | } |
1926 | ||
07e6de73 DP |
1927 | if (adreno_is_a690(adreno_gpu)) |
1928 | gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x90); | |
192f4ee3 | 1929 | /* Set dualQ + disable afull for A660 GPU */ |
07e6de73 | 1930 | else if (adreno_is_a660(adreno_gpu)) |
192f4ee3 | 1931 | gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x66906); |
af66706a KD |
1932 | else if (adreno_is_a7xx(adreno_gpu)) |
1933 | gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, | |
1934 | FIELD_PREP(GENMASK(19, 16), 6) | | |
1935 | FIELD_PREP(GENMASK(15, 12), 6) | | |
1936 | FIELD_PREP(GENMASK(11, 8), 9) | | |
1937 | BIT(3) | BIT(2) | | |
1938 | FIELD_PREP(GENMASK(1, 0), 2)); | |
192f4ee3 | 1939 | |
604234f3 JC |
1940 | /* Enable expanded apriv for targets that support it */ |
1941 | if (gpu->hw_apriv) { | |
af66706a KD |
1942 | if (adreno_is_a7xx(adreno_gpu)) { |
1943 | gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL, | |
1944 | A7XX_BR_APRIVMASK); | |
1945 | gpu_write(gpu, REG_A7XX_CP_BV_APRIV_CNTL, | |
1946 | A7XX_APRIV_MASK); | |
1947 | gpu_write(gpu, REG_A7XX_CP_LPAC_APRIV_CNTL, | |
1948 | A7XX_APRIV_MASK); | |
1949 | } else | |
1950 | gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL, | |
1951 | BIT(6) | BIT(5) | BIT(3) | BIT(2) | BIT(1)); | |
24e6938e JM |
1952 | } |
1953 | ||
4b565ca5 | 1954 | /* Enable interrupts */ |
af66706a KD |
1955 | gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, |
1956 | adreno_is_a7xx(adreno_gpu) ? A7XX_INT_MASK : A6XX_INT_MASK); | |
4b565ca5 JC |
1957 | |
1958 | ret = adreno_hw_init(gpu); | |
1959 | if (ret) | |
1960 | goto out; | |
1961 | ||
8ead9678 | 1962 | gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE, a6xx_gpu->sqe_iova); |
4b565ca5 | 1963 | |
f6828e0c | 1964 | /* Set the ringbuffer address */ |
cade05b2 | 1965 | gpu_write64(gpu, REG_A6XX_CP_RB_BASE, gpu->rb[0]->iova); |
f6828e0c | 1966 | |
d3a569fc JC |
1967 | /* Targets that support extended APRIV can use the RPTR shadow from |
1968 | * hardware but all the other ones need to disable the feature. Targets | |
1969 | * that support the WHERE_AM_I opcode can use that instead | |
1970 | */ | |
1971 | if (adreno_gpu->base.hw_apriv) | |
1972 | gpu_write(gpu, REG_A6XX_CP_RB_CNTL, MSM_GPU_RB_CNTL_DEFAULT); | |
1973 | else | |
1974 | gpu_write(gpu, REG_A6XX_CP_RB_CNTL, | |
1975 | MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE); | |
1976 | ||
8ead9678 RC |
1977 | /* Configure the RPTR shadow if needed: */ |
1978 | if (a6xx_gpu->shadow_bo) { | |
f73343fa | 1979 | gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR, |
d3a569fc JC |
1980 | shadowptr(a6xx_gpu, gpu->rb[0])); |
1981 | } | |
f6828e0c | 1982 | |
af66706a KD |
1983 | /* ..which means "always" on A7xx, also for BV shadow */ |
1984 | if (adreno_is_a7xx(adreno_gpu)) { | |
1985 | gpu_write64(gpu, REG_A7XX_CP_BV_RB_RPTR_ADDR, | |
1986 | rbmemptr(gpu->rb[0], bv_fence)); | |
1987 | } | |
1988 | ||
4b565ca5 JC |
1989 | /* Always come up on rb 0 */ |
1990 | a6xx_gpu->cur_ring = gpu->rb[0]; | |
1991 | ||
1d054c9b | 1992 | gpu->cur_ctx_seqno = 0; |
84c31ee1 | 1993 | |
4b565ca5 JC |
1994 | /* Enable the SQE_to start the CP engine */ |
1995 | gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1); | |
1996 | ||
af66706a | 1997 | ret = adreno_is_a7xx(adreno_gpu) ? a7xx_cp_init(gpu) : a6xx_cp_init(gpu); |
4b565ca5 JC |
1998 | if (ret) |
1999 | goto out; | |
2000 | ||
abccb9fe JC |
2001 | /* |
2002 | * Try to load a zap shader into the secure world. If successful | |
2003 | * we can use the CP to switch out of secure mode. If not then we | |
2004 | * have no resource but to try to switch ourselves out manually. If we | |
2005 | * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will | |
2006 | * be blocked and a permissions violation will soon follow. | |
2007 | */ | |
2008 | ret = a6xx_zap_shader_init(gpu); | |
2009 | if (!ret) { | |
2010 | OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1); | |
2011 | OUT_RING(gpu->rb[0], 0x00000000); | |
2012 | ||
2013 | a6xx_flush(gpu, gpu->rb[0]); | |
2014 | if (!a6xx_idle(gpu, gpu->rb[0])) | |
2015 | return -EINVAL; | |
15ab987c RC |
2016 | } else if (ret == -ENODEV) { |
2017 | /* | |
2018 | * This device does not use zap shader (but print a warning | |
2019 | * just in case someone got their dt wrong.. hopefully they | |
2020 | * have a debug UART to realize the error of their ways... | |
2021 | * if you mess this up you are about to crash horribly) | |
2022 | */ | |
abccb9fe JC |
2023 | dev_warn_once(gpu->dev->dev, |
2024 | "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n"); | |
2025 | gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0); | |
15273ffd | 2026 | ret = 0; |
15ab987c RC |
2027 | } else { |
2028 | return ret; | |
abccb9fe | 2029 | } |
4b565ca5 JC |
2030 | |
2031 | out: | |
5a903a44 KD |
2032 | if (adreno_has_gmu_wrapper(adreno_gpu)) |
2033 | return ret; | |
4b565ca5 JC |
2034 | /* |
2035 | * Tell the GMU that we are done touching the GPU and it can start power | |
2036 | * management | |
2037 | */ | |
2038 | a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); | |
2039 | ||
8167e6fa JM |
2040 | if (a6xx_gpu->gmu.legacy) { |
2041 | /* Take the GMU out of its special boot mode */ | |
2042 | a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER); | |
2043 | } | |
4b565ca5 JC |
2044 | |
2045 | return ret; | |
2046 | } | |
2047 | ||
f6f59072 RC |
2048 | static int a6xx_hw_init(struct msm_gpu *gpu) |
2049 | { | |
2050 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2051 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
2052 | int ret; | |
2053 | ||
2054 | mutex_lock(&a6xx_gpu->gmu.lock); | |
2055 | ret = hw_init(gpu); | |
2056 | mutex_unlock(&a6xx_gpu->gmu.lock); | |
2057 | ||
2058 | return ret; | |
2059 | } | |
2060 | ||
4b565ca5 JC |
2061 | static void a6xx_dump(struct msm_gpu *gpu) |
2062 | { | |
6a41da17 | 2063 | DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n", |
4b565ca5 JC |
2064 | gpu_read(gpu, REG_A6XX_RBBM_STATUS)); |
2065 | adreno_dump(gpu); | |
2066 | } | |
2067 | ||
4b565ca5 JC |
2068 | static void a6xx_recover(struct msm_gpu *gpu) |
2069 | { | |
2070 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2071 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
c11fa120 | 2072 | struct a6xx_gmu *gmu = &a6xx_gpu->gmu; |
f350bfb9 | 2073 | int i, active_submits; |
4b565ca5 JC |
2074 | |
2075 | adreno_dump_info(gpu); | |
2076 | ||
2077 | for (i = 0; i < 8; i++) | |
6a41da17 | 2078 | DRM_DEV_INFO(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i, |
4b565ca5 JC |
2079 | gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i))); |
2080 | ||
2081 | if (hang_debug) | |
2082 | a6xx_dump(gpu); | |
2083 | ||
f4a75b59 AO |
2084 | /* |
2085 | * To handle recovery specific sequences during the rpm suspend we are | |
2086 | * about to trigger | |
2087 | */ | |
2088 | a6xx_gpu->hung = true; | |
2089 | ||
3a9dd708 AO |
2090 | /* Halt SQE first */ |
2091 | gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 3); | |
2092 | ||
f350bfb9 AO |
2093 | pm_runtime_dont_use_autosuspend(&gpu->pdev->dev); |
2094 | ||
2095 | /* active_submit won't change until we make a submission */ | |
2096 | mutex_lock(&gpu->active_lock); | |
2097 | active_submits = gpu->active_submits; | |
2098 | ||
2099 | /* | |
2100 | * Temporarily clear active_submits count to silence a WARN() in the | |
2101 | * runtime suspend cb | |
2102 | */ | |
2103 | gpu->active_submits = 0; | |
2104 | ||
5a903a44 KD |
2105 | if (adreno_has_gmu_wrapper(adreno_gpu)) { |
2106 | /* Drain the outstanding traffic on memory buses */ | |
2107 | a6xx_bus_clear_pending_transactions(adreno_gpu, true); | |
2108 | ||
2109 | /* Reset the GPU to a clean state */ | |
2110 | a6xx_gpu_sw_reset(gpu, true); | |
2111 | a6xx_gpu_sw_reset(gpu, false); | |
2112 | } | |
2113 | ||
c11fa120 AO |
2114 | reinit_completion(&gmu->pd_gate); |
2115 | dev_pm_genpd_add_notifier(gmu->cxpd, &gmu->pd_nb); | |
2116 | dev_pm_genpd_synced_poweroff(gmu->cxpd); | |
2117 | ||
f350bfb9 AO |
2118 | /* Drop the rpm refcount from active submits */ |
2119 | if (active_submits) | |
2120 | pm_runtime_put(&gpu->pdev->dev); | |
2121 | ||
2122 | /* And the final one from recover worker */ | |
2123 | pm_runtime_put_sync(&gpu->pdev->dev); | |
2124 | ||
c11fa120 AO |
2125 | if (!wait_for_completion_timeout(&gmu->pd_gate, msecs_to_jiffies(1000))) |
2126 | DRM_DEV_ERROR(&gpu->pdev->dev, "cx gdsc didn't collapse\n"); | |
2127 | ||
2128 | dev_pm_genpd_remove_notifier(gmu->cxpd); | |
2129 | ||
f350bfb9 AO |
2130 | pm_runtime_use_autosuspend(&gpu->pdev->dev); |
2131 | ||
2132 | if (active_submits) | |
2133 | pm_runtime_get(&gpu->pdev->dev); | |
2134 | ||
2135 | pm_runtime_get_sync(&gpu->pdev->dev); | |
2136 | ||
2137 | gpu->active_submits = active_submits; | |
2138 | mutex_unlock(&gpu->active_lock); | |
4b565ca5 JC |
2139 | |
2140 | msm_gpu_hw_init(gpu); | |
f4a75b59 | 2141 | a6xx_gpu->hung = false; |
4b565ca5 JC |
2142 | } |
2143 | ||
2a574cc0 JC |
2144 | static const char *a6xx_uche_fault_block(struct msm_gpu *gpu, u32 mid) |
2145 | { | |
77beba37 | 2146 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); |
2a574cc0 JC |
2147 | static const char *uche_clients[7] = { |
2148 | "VFD", "SP", "VSC", "VPC", "HLSQ", "PC", "LRZ", | |
2149 | }; | |
2150 | u32 val; | |
2151 | ||
77beba37 CA |
2152 | if (adreno_is_a7xx(adreno_gpu)) { |
2153 | if (mid != 1 && mid != 2 && mid != 3 && mid != 8) | |
2154 | return "UNKNOWN"; | |
2155 | } else { | |
2156 | if (mid < 1 || mid > 3) | |
2157 | return "UNKNOWN"; | |
2158 | } | |
2a574cc0 JC |
2159 | |
2160 | /* | |
2161 | * The source of the data depends on the mid ID read from FSYNR1. | |
2162 | * and the client ID read from the UCHE block | |
2163 | */ | |
2164 | val = gpu_read(gpu, REG_A6XX_UCHE_CLIENT_PF); | |
2165 | ||
77beba37 CA |
2166 | if (adreno_is_a7xx(adreno_gpu)) { |
2167 | /* Bit 3 for mid=3 indicates BR or BV */ | |
2168 | static const char *uche_clients_a7xx[16] = { | |
2169 | "BR_VFD", "BR_SP", "BR_VSC", "BR_VPC", | |
2170 | "BR_HLSQ", "BR_PC", "BR_LRZ", "BR_TP", | |
2171 | "BV_VFD", "BV_SP", "BV_VSC", "BV_VPC", | |
2172 | "BV_HLSQ", "BV_PC", "BV_LRZ", "BV_TP", | |
2173 | }; | |
2174 | ||
2175 | /* LPAC has the same clients as BR and BV, but because it is | |
2176 | * compute-only some of them do not exist and there are holes | |
2177 | * in the array. | |
2178 | */ | |
2179 | static const char *uche_clients_lpac_a7xx[8] = { | |
2180 | "-", "LPAC_SP", "-", "-", | |
2181 | "LPAC_HLSQ", "-", "-", "LPAC_TP", | |
2182 | }; | |
2183 | ||
2184 | val &= GENMASK(6, 0); | |
2185 | ||
2186 | /* mid=3 refers to BR or BV */ | |
2187 | if (mid == 3) { | |
2188 | if (val < ARRAY_SIZE(uche_clients_a7xx)) | |
2189 | return uche_clients_a7xx[val]; | |
2190 | else | |
2191 | return "UCHE"; | |
2192 | } | |
2193 | ||
2194 | /* mid=8 refers to LPAC */ | |
2195 | if (mid == 8) { | |
2196 | if (val < ARRAY_SIZE(uche_clients_lpac_a7xx)) | |
2197 | return uche_clients_lpac_a7xx[val]; | |
2198 | else | |
2199 | return "UCHE_LPAC"; | |
2200 | } | |
2a574cc0 | 2201 | |
77beba37 CA |
2202 | /* mid=2 is a catchall for everything else in LPAC */ |
2203 | if (mid == 2) | |
2204 | return "UCHE_LPAC"; | |
2205 | ||
2206 | /* mid=1 is a catchall for everything else in BR/BV */ | |
2207 | return "UCHE"; | |
2208 | } else if (adreno_is_a660_family(adreno_gpu)) { | |
2209 | static const char *uche_clients_a660[8] = { | |
2210 | "VFD", "SP", "VSC", "VPC", "HLSQ", "PC", "LRZ", "TP", | |
2211 | }; | |
2a574cc0 | 2212 | |
77beba37 CA |
2213 | static const char *uche_clients_a660_not[8] = { |
2214 | "not VFD", "not SP", "not VSC", "not VPC", | |
2215 | "not HLSQ", "not PC", "not LRZ", "not TP", | |
2216 | }; | |
2217 | ||
2218 | val &= GENMASK(6, 0); | |
2219 | ||
2220 | if (mid == 3 && val < ARRAY_SIZE(uche_clients_a660)) | |
2221 | return uche_clients_a660[val]; | |
2222 | ||
2223 | if (mid == 1 && val < ARRAY_SIZE(uche_clients_a660_not)) | |
2224 | return uche_clients_a660_not[val]; | |
2225 | ||
2226 | return "UCHE"; | |
2227 | } else { | |
2228 | /* mid = 3 is most precise and refers to only one block per client */ | |
2229 | if (mid == 3) | |
2230 | return uche_clients[val & 7]; | |
2231 | ||
2232 | /* For mid=2 the source is TP or VFD except when the client id is 0 */ | |
2233 | if (mid == 2) | |
2234 | return ((val & 7) == 0) ? "TP" : "TP|VFD"; | |
2235 | ||
2236 | /* For mid=1 just return "UCHE" as a catchall for everything else */ | |
2237 | return "UCHE"; | |
2238 | } | |
2a574cc0 JC |
2239 | } |
2240 | ||
2241 | static const char *a6xx_fault_block(struct msm_gpu *gpu, u32 id) | |
2242 | { | |
77beba37 CA |
2243 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); |
2244 | ||
2a574cc0 JC |
2245 | if (id == 0) |
2246 | return "CP"; | |
2247 | else if (id == 4) | |
2248 | return "CCU"; | |
2249 | else if (id == 6) | |
2250 | return "CDP Prefetch"; | |
77beba37 CA |
2251 | else if (id == 7) |
2252 | return "GMU"; | |
2253 | else if (id == 5 && adreno_is_a7xx(adreno_gpu)) | |
2254 | return "Flag cache"; | |
2a574cc0 JC |
2255 | |
2256 | return a6xx_uche_fault_block(gpu, id); | |
2257 | } | |
2258 | ||
2a574cc0 | 2259 | static int a6xx_fault_handler(void *arg, unsigned long iova, int flags, void *data) |
4b565ca5 JC |
2260 | { |
2261 | struct msm_gpu *gpu = arg; | |
2a574cc0 | 2262 | struct adreno_smmu_fault_info *info = data; |
f62ad0f6 | 2263 | const char *block = "unknown"; |
4b565ca5 | 2264 | |
f62ad0f6 | 2265 | u32 scratch[] = { |
4b565ca5 JC |
2266 | gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)), |
2267 | gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)), | |
2268 | gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)), | |
f62ad0f6 DB |
2269 | gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)), |
2270 | }; | |
e25e92e0 | 2271 | |
f62ad0f6 DB |
2272 | if (info) |
2273 | block = a6xx_fault_block(gpu, info->fsynr1 & 0xff); | |
e25e92e0 | 2274 | |
f62ad0f6 | 2275 | return adreno_fault_handler(gpu, iova, flags, info, block, scratch); |
4b565ca5 JC |
2276 | } |
2277 | ||
2278 | static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu) | |
2279 | { | |
2280 | u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS); | |
2281 | ||
2282 | if (status & A6XX_CP_INT_CP_OPCODE_ERROR) { | |
2283 | u32 val; | |
2284 | ||
2285 | gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1); | |
2286 | val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA); | |
2287 | dev_err_ratelimited(&gpu->pdev->dev, | |
2288 | "CP | opcode error | possible opcode=0x%8.8X\n", | |
2289 | val); | |
2290 | } | |
2291 | ||
2292 | if (status & A6XX_CP_INT_CP_UCODE_ERROR) | |
2293 | dev_err_ratelimited(&gpu->pdev->dev, | |
2294 | "CP ucode error interrupt\n"); | |
2295 | ||
2296 | if (status & A6XX_CP_INT_CP_HW_FAULT_ERROR) | |
2297 | dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n", | |
2298 | gpu_read(gpu, REG_A6XX_CP_HW_FAULT)); | |
2299 | ||
2300 | if (status & A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) { | |
2301 | u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS); | |
2302 | ||
2303 | dev_err_ratelimited(&gpu->pdev->dev, | |
2304 | "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n", | |
2305 | val & (1 << 20) ? "READ" : "WRITE", | |
2306 | (val & 0x3ffff), val); | |
2307 | } | |
2308 | ||
af66706a | 2309 | if (status & A6XX_CP_INT_CP_AHB_ERROR && !adreno_is_a7xx(to_adreno_gpu(gpu))) |
4b565ca5 JC |
2310 | dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n"); |
2311 | ||
2312 | if (status & A6XX_CP_INT_CP_VSD_PARITY_ERROR) | |
2313 | dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n"); | |
2314 | ||
2315 | if (status & A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR) | |
2316 | dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n"); | |
2317 | ||
2318 | } | |
2319 | ||
2320 | static void a6xx_fault_detect_irq(struct msm_gpu *gpu) | |
2321 | { | |
2322 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2323 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
4b565ca5 JC |
2324 | struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); |
2325 | ||
e25e92e0 RC |
2326 | /* |
2327 | * If stalled on SMMU fault, we could trip the GPU's hang detection, | |
2328 | * but the fault handler will trigger the devcore dump, and we want | |
2329 | * to otherwise resume normally rather than killing the submit, so | |
2330 | * just bail. | |
2331 | */ | |
2332 | if (gpu_read(gpu, REG_A6XX_RBBM_STATUS3) & A6XX_RBBM_STATUS3_SMMU_STALLED_ON_FAULT) | |
2333 | return; | |
2334 | ||
4b565ca5 JC |
2335 | /* |
2336 | * Force the GPU to stay on until after we finish | |
2337 | * collecting information | |
2338 | */ | |
5a903a44 KD |
2339 | if (!adreno_has_gmu_wrapper(adreno_gpu)) |
2340 | gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1); | |
4b565ca5 JC |
2341 | |
2342 | DRM_DEV_ERROR(&gpu->pdev->dev, | |
2343 | "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n", | |
f9d5355f | 2344 | ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0, |
4b565ca5 JC |
2345 | gpu_read(gpu, REG_A6XX_RBBM_STATUS), |
2346 | gpu_read(gpu, REG_A6XX_CP_RB_RPTR), | |
2347 | gpu_read(gpu, REG_A6XX_CP_RB_WPTR), | |
cade05b2 | 2348 | gpu_read64(gpu, REG_A6XX_CP_IB1_BASE), |
4b565ca5 | 2349 | gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE), |
cade05b2 | 2350 | gpu_read64(gpu, REG_A6XX_CP_IB2_BASE), |
4b565ca5 JC |
2351 | gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE)); |
2352 | ||
2353 | /* Turn off the hangcheck timer to keep it from bothering us */ | |
2354 | del_timer(&gpu->hangcheck_timer); | |
2355 | ||
7e688294 | 2356 | kthread_queue_work(gpu->worker, &gpu->recover_work); |
4b565ca5 JC |
2357 | } |
2358 | ||
2359 | static irqreturn_t a6xx_irq(struct msm_gpu *gpu) | |
2360 | { | |
5edf2750 | 2361 | struct msm_drm_private *priv = gpu->dev->dev_private; |
4b565ca5 JC |
2362 | u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS); |
2363 | ||
2364 | gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status); | |
2365 | ||
5edf2750 RC |
2366 | if (priv->disable_err_irq) |
2367 | status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS; | |
2368 | ||
4b565ca5 JC |
2369 | if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT) |
2370 | a6xx_fault_detect_irq(gpu); | |
2371 | ||
2372 | if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR) | |
2373 | dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n"); | |
2374 | ||
2375 | if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR) | |
2376 | a6xx_cp_hw_err_irq(gpu); | |
2377 | ||
2378 | if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW) | |
2379 | dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n"); | |
2380 | ||
2381 | if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW) | |
2382 | dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n"); | |
2383 | ||
2384 | if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS) | |
2385 | dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n"); | |
2386 | ||
2387 | if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) | |
2388 | msm_gpu_retire(gpu); | |
2389 | ||
2390 | return IRQ_HANDLED; | |
2391 | } | |
2392 | ||
474dadb8 SM |
2393 | static void a6xx_llc_deactivate(struct a6xx_gpu *a6xx_gpu) |
2394 | { | |
2395 | llcc_slice_deactivate(a6xx_gpu->llc_slice); | |
2396 | llcc_slice_deactivate(a6xx_gpu->htw_llc_slice); | |
2397 | } | |
2398 | ||
2399 | static void a6xx_llc_activate(struct a6xx_gpu *a6xx_gpu) | |
2400 | { | |
3d247123 JC |
2401 | struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; |
2402 | struct msm_gpu *gpu = &adreno_gpu->base; | |
9ba873e6 | 2403 | u32 cntl1_regval = 0; |
474dadb8 SM |
2404 | |
2405 | if (IS_ERR(a6xx_gpu->llc_mmio)) | |
2406 | return; | |
2407 | ||
2408 | if (!llcc_slice_activate(a6xx_gpu->llc_slice)) { | |
9ba873e6 | 2409 | u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice); |
474dadb8 SM |
2410 | |
2411 | gpu_scid &= 0x1f; | |
2412 | cntl1_regval = (gpu_scid << 0) | (gpu_scid << 5) | (gpu_scid << 10) | | |
2413 | (gpu_scid << 15) | (gpu_scid << 20); | |
9ba873e6 AO |
2414 | |
2415 | /* On A660, the SCID programming for UCHE traffic is done in | |
2416 | * A6XX_GBIF_SCACHE_CNTL0[14:10] | |
2417 | */ | |
2418 | if (adreno_is_a660_family(adreno_gpu)) | |
2419 | gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL0, (0x1f << 10) | | |
2420 | (1 << 8), (gpu_scid << 10) | (1 << 8)); | |
474dadb8 SM |
2421 | } |
2422 | ||
3d247123 JC |
2423 | /* |
2424 | * For targets with a MMU500, activate the slice but don't program the | |
2425 | * register. The XBL will take care of that. | |
2426 | */ | |
474dadb8 | 2427 | if (!llcc_slice_activate(a6xx_gpu->htw_llc_slice)) { |
3d247123 JC |
2428 | if (!a6xx_gpu->have_mmu500) { |
2429 | u32 gpuhtw_scid = llcc_get_slice_id(a6xx_gpu->htw_llc_slice); | |
474dadb8 | 2430 | |
3d247123 JC |
2431 | gpuhtw_scid &= 0x1f; |
2432 | cntl1_regval |= FIELD_PREP(GENMASK(29, 25), gpuhtw_scid); | |
2433 | } | |
474dadb8 SM |
2434 | } |
2435 | ||
a6f24383 AO |
2436 | if (!cntl1_regval) |
2437 | return; | |
2438 | ||
2439 | /* | |
2440 | * Program the slice IDs for the various GPU blocks and GPU MMU | |
2441 | * pagetables | |
2442 | */ | |
2443 | if (!a6xx_gpu->have_mmu500) { | |
2444 | a6xx_llc_write(a6xx_gpu, | |
2445 | REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_1, cntl1_regval); | |
2446 | ||
474dadb8 | 2447 | /* |
a6f24383 AO |
2448 | * Program cacheability overrides to not allocate cache |
2449 | * lines on a write miss | |
474dadb8 | 2450 | */ |
a6f24383 AO |
2451 | a6xx_llc_rmw(a6xx_gpu, |
2452 | REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_0, 0xF, 0x03); | |
2453 | return; | |
474dadb8 | 2454 | } |
a6f24383 AO |
2455 | |
2456 | gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL1, GENMASK(24, 0), cntl1_regval); | |
474dadb8 SM |
2457 | } |
2458 | ||
af66706a KD |
2459 | static void a7xx_llc_activate(struct a6xx_gpu *a6xx_gpu) |
2460 | { | |
2461 | struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; | |
2462 | struct msm_gpu *gpu = &adreno_gpu->base; | |
2463 | ||
2464 | if (IS_ERR(a6xx_gpu->llc_mmio)) | |
2465 | return; | |
2466 | ||
2467 | if (!llcc_slice_activate(a6xx_gpu->llc_slice)) { | |
2468 | u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice); | |
2469 | ||
2470 | gpu_scid &= GENMASK(4, 0); | |
2471 | ||
2472 | gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL1, | |
2473 | FIELD_PREP(GENMASK(29, 25), gpu_scid) | | |
2474 | FIELD_PREP(GENMASK(24, 20), gpu_scid) | | |
2475 | FIELD_PREP(GENMASK(19, 15), gpu_scid) | | |
2476 | FIELD_PREP(GENMASK(14, 10), gpu_scid) | | |
2477 | FIELD_PREP(GENMASK(9, 5), gpu_scid) | | |
2478 | FIELD_PREP(GENMASK(4, 0), gpu_scid)); | |
2479 | ||
2480 | gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL0, | |
2481 | FIELD_PREP(GENMASK(14, 10), gpu_scid) | | |
2482 | BIT(8)); | |
2483 | } | |
2484 | ||
2485 | llcc_slice_activate(a6xx_gpu->htw_llc_slice); | |
2486 | } | |
2487 | ||
474dadb8 SM |
2488 | static void a6xx_llc_slices_destroy(struct a6xx_gpu *a6xx_gpu) |
2489 | { | |
5a903a44 KD |
2490 | /* No LLCC on non-RPMh (and by extension, non-GMU) SoCs */ |
2491 | if (adreno_has_gmu_wrapper(&a6xx_gpu->base)) | |
2492 | return; | |
2493 | ||
474dadb8 SM |
2494 | llcc_slice_putd(a6xx_gpu->llc_slice); |
2495 | llcc_slice_putd(a6xx_gpu->htw_llc_slice); | |
2496 | } | |
2497 | ||
2498 | static void a6xx_llc_slices_init(struct platform_device *pdev, | |
af66706a | 2499 | struct a6xx_gpu *a6xx_gpu, bool is_a7xx) |
474dadb8 | 2500 | { |
3d247123 JC |
2501 | struct device_node *phandle; |
2502 | ||
5a903a44 KD |
2503 | /* No LLCC on non-RPMh (and by extension, non-GMU) SoCs */ |
2504 | if (adreno_has_gmu_wrapper(&a6xx_gpu->base)) | |
2505 | return; | |
2506 | ||
3d247123 | 2507 | /* |
af66706a KD |
2508 | * There is a different programming path for A6xx targets with an |
2509 | * mmu500 attached, so detect if that is the case | |
3d247123 JC |
2510 | */ |
2511 | phandle = of_parse_phandle(pdev->dev.of_node, "iommus", 0); | |
2512 | a6xx_gpu->have_mmu500 = (phandle && | |
2513 | of_device_is_compatible(phandle, "arm,mmu-500")); | |
2514 | of_node_put(phandle); | |
2515 | ||
af66706a | 2516 | if (is_a7xx || !a6xx_gpu->have_mmu500) |
c0e745d7 | 2517 | a6xx_gpu->llc_mmio = msm_ioremap(pdev, "cx_mem"); |
af66706a KD |
2518 | else |
2519 | a6xx_gpu->llc_mmio = NULL; | |
4b95d371 | 2520 | |
474dadb8 SM |
2521 | a6xx_gpu->llc_slice = llcc_slice_getd(LLCC_GPU); |
2522 | a6xx_gpu->htw_llc_slice = llcc_slice_getd(LLCC_GPUHTW); | |
2523 | ||
276619c0 | 2524 | if (IS_ERR_OR_NULL(a6xx_gpu->llc_slice) && IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice)) |
474dadb8 SM |
2525 | a6xx_gpu->llc_mmio = ERR_PTR(-EINVAL); |
2526 | } | |
2527 | ||
3773a57d KD |
2528 | #define GBIF_CLIENT_HALT_MASK BIT(0) |
2529 | #define GBIF_ARB_HALT_MASK BIT(1) | |
2530 | #define VBIF_XIN_HALT_CTRL0_MASK GENMASK(3, 0) | |
8296ff0a KD |
2531 | #define VBIF_RESET_ACK_MASK 0xF0 |
2532 | #define GPR0_GBIF_HALT_REQUEST 0x1E0 | |
6e332c99 KD |
2533 | |
2534 | void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off) | |
2535 | { | |
2536 | struct msm_gpu *gpu = &adreno_gpu->base; | |
2537 | ||
8296ff0a KD |
2538 | if (adreno_is_a619_holi(adreno_gpu)) { |
2539 | gpu_write(gpu, REG_A6XX_RBBM_GPR0_CNTL, GPR0_GBIF_HALT_REQUEST); | |
2540 | spin_until((gpu_read(gpu, REG_A6XX_RBBM_VBIF_GX_RESET_STATUS) & | |
2541 | (VBIF_RESET_ACK_MASK)) == VBIF_RESET_ACK_MASK); | |
2542 | } else if (!a6xx_has_gbif(adreno_gpu)) { | |
3773a57d | 2543 | gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, VBIF_XIN_HALT_CTRL0_MASK); |
6e332c99 | 2544 | spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & |
3773a57d | 2545 | (VBIF_XIN_HALT_CTRL0_MASK)) == VBIF_XIN_HALT_CTRL0_MASK); |
6e332c99 KD |
2546 | gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0); |
2547 | ||
2548 | return; | |
2549 | } | |
2550 | ||
2551 | if (gx_off) { | |
2552 | /* Halt the gx side of GBIF */ | |
2553 | gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 1); | |
2554 | spin_until(gpu_read(gpu, REG_A6XX_RBBM_GBIF_HALT_ACK) & 1); | |
2555 | } | |
2556 | ||
2557 | /* Halt new client requests on GBIF */ | |
2558 | gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK); | |
2559 | spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & | |
2560 | (GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK); | |
2561 | ||
2562 | /* Halt all AXI requests on GBIF */ | |
2563 | gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK); | |
2564 | spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & | |
2565 | (GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK); | |
2566 | ||
2567 | /* The GBIF halt needs to be explicitly cleared */ | |
2568 | gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0); | |
2569 | } | |
2570 | ||
277b9678 KD |
2571 | void a6xx_gpu_sw_reset(struct msm_gpu *gpu, bool assert) |
2572 | { | |
e7fc9398 KD |
2573 | /* 11nm chips (e.g. ones with A610) have hw issues with the reset line! */ |
2574 | if (adreno_is_a610(to_adreno_gpu(gpu))) | |
2575 | return; | |
2576 | ||
277b9678 KD |
2577 | gpu_write(gpu, REG_A6XX_RBBM_SW_RESET_CMD, assert); |
2578 | /* Perform a bogus read and add a brief delay to ensure ordering. */ | |
2579 | gpu_read(gpu, REG_A6XX_RBBM_SW_RESET_CMD); | |
2580 | udelay(1); | |
2581 | ||
2582 | /* The reset line needs to be asserted for at least 100 us */ | |
2583 | if (assert) | |
2584 | udelay(100); | |
2585 | } | |
2586 | ||
5a903a44 | 2587 | static int a6xx_gmu_pm_resume(struct msm_gpu *gpu) |
4b565ca5 JC |
2588 | { |
2589 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2590 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
2591 | int ret; | |
2592 | ||
4b565ca5 JC |
2593 | gpu->needs_hw_init = true; |
2594 | ||
ec1cb6e4 RC |
2595 | trace_msm_gpu_resume(0); |
2596 | ||
f6f59072 | 2597 | mutex_lock(&a6xx_gpu->gmu.lock); |
41570b74 | 2598 | ret = a6xx_gmu_resume(a6xx_gpu); |
f6f59072 | 2599 | mutex_unlock(&a6xx_gpu->gmu.lock); |
41570b74 JC |
2600 | if (ret) |
2601 | return ret; | |
2602 | ||
af5b4fff | 2603 | msm_devfreq_resume(gpu); |
a2c3c0a5 | 2604 | |
0776ad92 | 2605 | adreno_is_a7xx(adreno_gpu) ? a7xx_llc_activate(a6xx_gpu) : a6xx_llc_activate(a6xx_gpu); |
474dadb8 | 2606 | |
5a903a44 | 2607 | return ret; |
4b565ca5 JC |
2608 | } |
2609 | ||
5a903a44 KD |
2610 | static int a6xx_pm_resume(struct msm_gpu *gpu) |
2611 | { | |
2612 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2613 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
2614 | struct a6xx_gmu *gmu = &a6xx_gpu->gmu; | |
2615 | unsigned long freq = gpu->fast_rate; | |
2616 | struct dev_pm_opp *opp; | |
2617 | int ret; | |
2618 | ||
2619 | gpu->needs_hw_init = true; | |
2620 | ||
2621 | trace_msm_gpu_resume(0); | |
2622 | ||
2623 | mutex_lock(&a6xx_gpu->gmu.lock); | |
2624 | ||
2625 | opp = dev_pm_opp_find_freq_ceil(&gpu->pdev->dev, &freq); | |
2626 | if (IS_ERR(opp)) { | |
2627 | ret = PTR_ERR(opp); | |
2628 | goto err_set_opp; | |
2629 | } | |
2630 | dev_pm_opp_put(opp); | |
2631 | ||
2632 | /* Set the core clock and bus bw, having VDD scaling in mind */ | |
2633 | dev_pm_opp_set_opp(&gpu->pdev->dev, opp); | |
2634 | ||
2635 | pm_runtime_resume_and_get(gmu->dev); | |
2636 | pm_runtime_resume_and_get(gmu->gxpd); | |
2637 | ||
2638 | ret = clk_bulk_prepare_enable(gpu->nr_clocks, gpu->grp_clks); | |
2639 | if (ret) | |
2640 | goto err_bulk_clk; | |
2641 | ||
8296ff0a KD |
2642 | if (adreno_is_a619_holi(adreno_gpu)) |
2643 | a6xx_sptprac_enable(gmu); | |
2644 | ||
5a903a44 KD |
2645 | /* If anything goes south, tear the GPU down piece by piece.. */ |
2646 | if (ret) { | |
2647 | err_bulk_clk: | |
2648 | pm_runtime_put(gmu->gxpd); | |
2649 | pm_runtime_put(gmu->dev); | |
2650 | dev_pm_opp_set_opp(&gpu->pdev->dev, NULL); | |
2651 | } | |
2652 | err_set_opp: | |
2653 | mutex_unlock(&a6xx_gpu->gmu.lock); | |
2654 | ||
2655 | if (!ret) | |
2656 | msm_devfreq_resume(gpu); | |
2657 | ||
2658 | return ret; | |
2659 | } | |
2660 | ||
2661 | static int a6xx_gmu_pm_suspend(struct msm_gpu *gpu) | |
4b565ca5 JC |
2662 | { |
2663 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2664 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
e8b0b994 | 2665 | int i, ret; |
4b565ca5 | 2666 | |
ec1cb6e4 RC |
2667 | trace_msm_gpu_suspend(0); |
2668 | ||
474dadb8 SM |
2669 | a6xx_llc_deactivate(a6xx_gpu); |
2670 | ||
af5b4fff | 2671 | msm_devfreq_suspend(gpu); |
a2c3c0a5 | 2672 | |
f6f59072 | 2673 | mutex_lock(&a6xx_gpu->gmu.lock); |
e8b0b994 | 2674 | ret = a6xx_gmu_stop(a6xx_gpu); |
f6f59072 | 2675 | mutex_unlock(&a6xx_gpu->gmu.lock); |
e8b0b994 RC |
2676 | if (ret) |
2677 | return ret; | |
2678 | ||
ce86c239 | 2679 | if (a6xx_gpu->shadow_bo) |
e8b0b994 RC |
2680 | for (i = 0; i < gpu->nr_rings; i++) |
2681 | a6xx_gpu->shadow[i] = 0; | |
2682 | ||
860a7b2a RC |
2683 | gpu->suspend_count++; |
2684 | ||
e8b0b994 | 2685 | return 0; |
4b565ca5 JC |
2686 | } |
2687 | ||
5a903a44 KD |
2688 | static int a6xx_pm_suspend(struct msm_gpu *gpu) |
2689 | { | |
2690 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2691 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
2692 | struct a6xx_gmu *gmu = &a6xx_gpu->gmu; | |
2693 | int i; | |
2694 | ||
2695 | trace_msm_gpu_suspend(0); | |
2696 | ||
2697 | msm_devfreq_suspend(gpu); | |
2698 | ||
2699 | mutex_lock(&a6xx_gpu->gmu.lock); | |
2700 | ||
2701 | /* Drain the outstanding traffic on memory buses */ | |
2702 | a6xx_bus_clear_pending_transactions(adreno_gpu, true); | |
2703 | ||
8296ff0a KD |
2704 | if (adreno_is_a619_holi(adreno_gpu)) |
2705 | a6xx_sptprac_disable(gmu); | |
2706 | ||
5a903a44 KD |
2707 | clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks); |
2708 | ||
2709 | pm_runtime_put_sync(gmu->gxpd); | |
2710 | dev_pm_opp_set_opp(&gpu->pdev->dev, NULL); | |
2711 | pm_runtime_put_sync(gmu->dev); | |
2712 | ||
2713 | mutex_unlock(&a6xx_gpu->gmu.lock); | |
2714 | ||
2715 | if (a6xx_gpu->shadow_bo) | |
2716 | for (i = 0; i < gpu->nr_rings; i++) | |
2717 | a6xx_gpu->shadow[i] = 0; | |
2718 | ||
2719 | gpu->suspend_count++; | |
2720 | ||
2721 | return 0; | |
2722 | } | |
2723 | ||
2724 | static int a6xx_gmu_get_timestamp(struct msm_gpu *gpu, uint64_t *value) | |
4b565ca5 JC |
2725 | { |
2726 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2727 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
5f98b33b | 2728 | |
f6f59072 | 2729 | mutex_lock(&a6xx_gpu->gmu.lock); |
4b565ca5 JC |
2730 | |
2731 | /* Force the GPU power on so we can read this register */ | |
7a7cbf2a | 2732 | a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET); |
4b565ca5 | 2733 | |
f73343fa | 2734 | *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER); |
4b565ca5 | 2735 | |
7a7cbf2a | 2736 | a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET); |
f6f59072 RC |
2737 | |
2738 | mutex_unlock(&a6xx_gpu->gmu.lock); | |
2739 | ||
4b565ca5 JC |
2740 | return 0; |
2741 | } | |
2742 | ||
5a903a44 KD |
2743 | static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value) |
2744 | { | |
2745 | *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER); | |
2746 | return 0; | |
2747 | } | |
2748 | ||
4b565ca5 JC |
2749 | static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu) |
2750 | { | |
2751 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2752 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
2753 | ||
2754 | return a6xx_gpu->cur_ring; | |
2755 | } | |
2756 | ||
2757 | static void a6xx_destroy(struct msm_gpu *gpu) | |
2758 | { | |
2759 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2760 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
2761 | ||
2762 | if (a6xx_gpu->sqe_bo) { | |
7ad0e8cf | 2763 | msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace); |
f7d33950 | 2764 | drm_gem_object_put(a6xx_gpu->sqe_bo); |
4b565ca5 JC |
2765 | } |
2766 | ||
d3a569fc JC |
2767 | if (a6xx_gpu->shadow_bo) { |
2768 | msm_gem_unpin_iova(a6xx_gpu->shadow_bo, gpu->aspace); | |
2769 | drm_gem_object_put(a6xx_gpu->shadow_bo); | |
2770 | } | |
2771 | ||
474dadb8 SM |
2772 | a6xx_llc_slices_destroy(a6xx_gpu); |
2773 | ||
4b565ca5 JC |
2774 | a6xx_gmu_remove(a6xx_gpu); |
2775 | ||
2776 | adreno_gpu_cleanup(adreno_gpu); | |
fe7952c6 | 2777 | |
4b565ca5 JC |
2778 | kfree(a6xx_gpu); |
2779 | } | |
2780 | ||
15c41198 | 2781 | static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate) |
a2c3c0a5 SM |
2782 | { |
2783 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2784 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
15c41198 | 2785 | u64 busy_cycles; |
a2c3c0a5 | 2786 | |
15c41198 CW |
2787 | /* 19.2MHz */ |
2788 | *out_sample_rate = 19200000; | |
eadf7928 | 2789 | |
a2c3c0a5 SM |
2790 | busy_cycles = gmu_read64(&a6xx_gpu->gmu, |
2791 | REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L, | |
2792 | REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H); | |
2793 | ||
15c41198 | 2794 | return busy_cycles; |
a2c3c0a5 SM |
2795 | } |
2796 | ||
6694482a DA |
2797 | static void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp, |
2798 | bool suspended) | |
f6f59072 RC |
2799 | { |
2800 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2801 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
2802 | ||
2803 | mutex_lock(&a6xx_gpu->gmu.lock); | |
6694482a | 2804 | a6xx_gmu_set_freq(gpu, opp, suspended); |
f6f59072 RC |
2805 | mutex_unlock(&a6xx_gpu->gmu.lock); |
2806 | } | |
2807 | ||
45596f25 SPR |
2808 | static struct msm_gem_address_space * |
2809 | a6xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev) | |
2810 | { | |
2811 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2812 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
3236130b | 2813 | unsigned long quirks = 0; |
45596f25 SPR |
2814 | |
2815 | /* | |
2816 | * This allows GPU to set the bus attributes required to use system | |
2817 | * cache on behalf of the iommu page table walker. | |
2818 | */ | |
38e27a6f DB |
2819 | if (!IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice) && |
2820 | !device_iommu_capable(&pdev->dev, IOMMU_CAP_CACHE_COHERENCY)) | |
3236130b | 2821 | quirks |= IO_PGTABLE_QUIRK_ARM_OUTER_WBWA; |
45596f25 | 2822 | |
822ff993 | 2823 | return adreno_iommu_create_address_space(gpu, pdev, quirks); |
45596f25 SPR |
2824 | } |
2825 | ||
84c31ee1 JC |
2826 | static struct msm_gem_address_space * |
2827 | a6xx_create_private_address_space(struct msm_gpu *gpu) | |
2828 | { | |
2829 | struct msm_mmu *mmu; | |
2830 | ||
2831 | mmu = msm_iommu_pagetable_create(gpu->aspace->mmu); | |
2832 | ||
2833 | if (IS_ERR(mmu)) | |
2834 | return ERR_CAST(mmu); | |
2835 | ||
2836 | return msm_gem_address_space_create(mmu, | |
36bbfdb8 RC |
2837 | "gpu", 0x100000000ULL, |
2838 | adreno_private_address_space_size(gpu)); | |
84c31ee1 JC |
2839 | } |
2840 | ||
d3a569fc JC |
2841 | static uint32_t a6xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) |
2842 | { | |
2843 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); | |
2844 | struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); | |
2845 | ||
2846 | if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) | |
2847 | return a6xx_gpu->shadow[ring->id]; | |
2848 | ||
2849 | return ring->memptrs->rptr = gpu_read(gpu, REG_A6XX_CP_RB_RPTR); | |
2850 | } | |
2851 | ||
d73b1d02 RC |
2852 | static bool a6xx_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring) |
2853 | { | |
2854 | struct msm_cp_state cp_state = { | |
2855 | .ib1_base = gpu_read64(gpu, REG_A6XX_CP_IB1_BASE), | |
2856 | .ib2_base = gpu_read64(gpu, REG_A6XX_CP_IB2_BASE), | |
2857 | .ib1_rem = gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE), | |
2858 | .ib2_rem = gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE), | |
2859 | }; | |
2860 | bool progress; | |
2861 | ||
2862 | /* | |
2863 | * Adjust the remaining data to account for what has already been | |
2864 | * fetched from memory, but not yet consumed by the SQE. | |
2865 | * | |
2866 | * This is not *technically* correct, the amount buffered could | |
2867 | * exceed the IB size due to hw prefetching ahead, but: | |
2868 | * | |
2869 | * (1) We aren't trying to find the exact position, just whether | |
2870 | * progress has been made | |
2871 | * (2) The CP_REG_TO_MEM at the end of a submit should be enough | |
2872 | * to prevent prefetching into an unrelated submit. (And | |
2873 | * either way, at some point the ROQ will be full.) | |
2874 | */ | |
f73343fa RC |
2875 | cp_state.ib1_rem += gpu_read(gpu, REG_A6XX_CP_ROQ_AVAIL_IB1) >> 16; |
2876 | cp_state.ib2_rem += gpu_read(gpu, REG_A6XX_CP_ROQ_AVAIL_IB2) >> 16; | |
d73b1d02 RC |
2877 | |
2878 | progress = !!memcmp(&cp_state, &ring->last_cp_state, sizeof(cp_state)); | |
2879 | ||
2880 | ring->last_cp_state = cp_state; | |
2881 | ||
2882 | return progress; | |
2883 | } | |
2884 | ||
c928a05e | 2885 | static u32 fuse_to_supp_hw(const struct adreno_info *info, u32 fuse) |
cd036d54 | 2886 | { |
c928a05e RC |
2887 | if (!info->speedbins) |
2888 | return UINT_MAX; | |
63899a73 | 2889 | |
c928a05e RC |
2890 | for (int i = 0; info->speedbins[i].fuse != SHRT_MAX; i++) |
2891 | if (info->speedbins[i].fuse == fuse) | |
2892 | return BIT(info->speedbins[i].speedbin); | |
c43de1aa AO |
2893 | |
2894 | return UINT_MAX; | |
2895 | } | |
2896 | ||
c928a05e | 2897 | static int a6xx_set_supported_hw(struct device *dev, const struct adreno_info *info) |
fe7952c6 | 2898 | { |
f6d19187 | 2899 | u32 supp_hw; |
c9f737c7 | 2900 | u32 speedbin; |
7bf168c8 | 2901 | int ret; |
fe7952c6 | 2902 | |
afab9d91 | 2903 | ret = adreno_read_speedbin(dev, &speedbin); |
2b0b219e JS |
2904 | /* |
2905 | * -ENOENT means that the platform doesn't support speedbin which is | |
2906 | * fine | |
2907 | */ | |
2908 | if (ret == -ENOENT) { | |
2909 | return 0; | |
2910 | } else if (ret) { | |
f6d19187 RC |
2911 | dev_err_probe(dev, ret, |
2912 | "failed to read speed-bin. Some OPPs may not be supported by hardware\n"); | |
2913 | return ret; | |
fe7952c6 AO |
2914 | } |
2915 | ||
c928a05e RC |
2916 | supp_hw = fuse_to_supp_hw(info, speedbin); |
2917 | ||
2918 | if (supp_hw == UINT_MAX) { | |
2919 | DRM_DEV_ERROR(dev, | |
2920 | "missing support for speed-bin: %u. Some OPPs may not be supported by hardware\n", | |
2921 | speedbin); | |
75cb60d4 | 2922 | supp_hw = BIT(0); /* Default */ |
c928a05e | 2923 | } |
fe7952c6 | 2924 | |
11120e93 YL |
2925 | ret = devm_pm_opp_set_supported_hw(dev, &supp_hw, 1); |
2926 | if (ret) | |
2927 | return ret; | |
fe7952c6 | 2928 | |
fe7952c6 AO |
2929 | return 0; |
2930 | } | |
2931 | ||
4b565ca5 JC |
2932 | static const struct adreno_gpu_funcs funcs = { |
2933 | .base = { | |
2934 | .get_param = adreno_get_param, | |
f7ddbf55 | 2935 | .set_param = adreno_set_param, |
4b565ca5 | 2936 | .hw_init = a6xx_hw_init, |
8ead9678 | 2937 | .ucode_load = a6xx_ucode_load, |
5a903a44 KD |
2938 | .pm_suspend = a6xx_gmu_pm_suspend, |
2939 | .pm_resume = a6xx_gmu_pm_resume, | |
4b565ca5 JC |
2940 | .recover = a6xx_recover, |
2941 | .submit = a6xx_submit, | |
4b565ca5 JC |
2942 | .active_ring = a6xx_active_ring, |
2943 | .irq = a6xx_irq, | |
2944 | .destroy = a6xx_destroy, | |
b02872df | 2945 | #if defined(CONFIG_DRM_MSM_GPU_STATE) |
4b565ca5 JC |
2946 | .show = a6xx_show, |
2947 | #endif | |
a2c3c0a5 SM |
2948 | .gpu_busy = a6xx_gpu_busy, |
2949 | .gpu_get_freq = a6xx_gmu_get_freq, | |
f6f59072 | 2950 | .gpu_set_freq = a6xx_gpu_set_freq, |
b02872df | 2951 | #if defined(CONFIG_DRM_MSM_GPU_STATE) |
1707add8 JC |
2952 | .gpu_state_get = a6xx_gpu_state_get, |
2953 | .gpu_state_put = a6xx_gpu_state_put, | |
5a903a44 KD |
2954 | #endif |
2955 | .create_address_space = a6xx_create_address_space, | |
2956 | .create_private_address_space = a6xx_create_private_address_space, | |
2957 | .get_rptr = a6xx_get_rptr, | |
2958 | .progress = a6xx_progress, | |
2959 | }, | |
2960 | .get_timestamp = a6xx_gmu_get_timestamp, | |
2961 | }; | |
2962 | ||
2963 | static const struct adreno_gpu_funcs funcs_gmuwrapper = { | |
2964 | .base = { | |
2965 | .get_param = adreno_get_param, | |
2966 | .set_param = adreno_set_param, | |
2967 | .hw_init = a6xx_hw_init, | |
2968 | .ucode_load = a6xx_ucode_load, | |
2969 | .pm_suspend = a6xx_pm_suspend, | |
2970 | .pm_resume = a6xx_pm_resume, | |
2971 | .recover = a6xx_recover, | |
2972 | .submit = a6xx_submit, | |
2973 | .active_ring = a6xx_active_ring, | |
2974 | .irq = a6xx_irq, | |
2975 | .destroy = a6xx_destroy, | |
2976 | #if defined(CONFIG_DRM_MSM_GPU_STATE) | |
2977 | .show = a6xx_show, | |
2978 | #endif | |
2979 | .gpu_busy = a6xx_gpu_busy, | |
2980 | #if defined(CONFIG_DRM_MSM_GPU_STATE) | |
2981 | .gpu_state_get = a6xx_gpu_state_get, | |
2982 | .gpu_state_put = a6xx_gpu_state_put, | |
b02872df | 2983 | #endif |
45596f25 | 2984 | .create_address_space = a6xx_create_address_space, |
84c31ee1 | 2985 | .create_private_address_space = a6xx_create_private_address_space, |
d3a569fc | 2986 | .get_rptr = a6xx_get_rptr, |
d73b1d02 | 2987 | .progress = a6xx_progress, |
4b565ca5 JC |
2988 | }, |
2989 | .get_timestamp = a6xx_get_timestamp, | |
2990 | }; | |
2991 | ||
af66706a KD |
2992 | static const struct adreno_gpu_funcs funcs_a7xx = { |
2993 | .base = { | |
2994 | .get_param = adreno_get_param, | |
2995 | .set_param = adreno_set_param, | |
2996 | .hw_init = a6xx_hw_init, | |
2997 | .ucode_load = a6xx_ucode_load, | |
2998 | .pm_suspend = a6xx_gmu_pm_suspend, | |
2999 | .pm_resume = a6xx_gmu_pm_resume, | |
3000 | .recover = a6xx_recover, | |
3001 | .submit = a7xx_submit, | |
3002 | .active_ring = a6xx_active_ring, | |
3003 | .irq = a6xx_irq, | |
3004 | .destroy = a6xx_destroy, | |
3005 | #if defined(CONFIG_DRM_MSM_GPU_STATE) | |
3006 | .show = a6xx_show, | |
3007 | #endif | |
3008 | .gpu_busy = a6xx_gpu_busy, | |
3009 | .gpu_get_freq = a6xx_gmu_get_freq, | |
3010 | .gpu_set_freq = a6xx_gpu_set_freq, | |
3011 | #if defined(CONFIG_DRM_MSM_GPU_STATE) | |
3012 | .gpu_state_get = a6xx_gpu_state_get, | |
3013 | .gpu_state_put = a6xx_gpu_state_put, | |
3014 | #endif | |
3015 | .create_address_space = a6xx_create_address_space, | |
3016 | .create_private_address_space = a6xx_create_private_address_space, | |
3017 | .get_rptr = a6xx_get_rptr, | |
3018 | .progress = a6xx_progress, | |
3019 | }, | |
3020 | .get_timestamp = a6xx_gmu_get_timestamp, | |
3021 | }; | |
3022 | ||
4b565ca5 JC |
3023 | struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) |
3024 | { | |
3025 | struct msm_drm_private *priv = dev->dev_private; | |
3026 | struct platform_device *pdev = priv->gpu_pdev; | |
e9ba8d55 | 3027 | struct adreno_platform_config *config = pdev->dev.platform_data; |
4b565ca5 JC |
3028 | struct device_node *node; |
3029 | struct a6xx_gpu *a6xx_gpu; | |
3030 | struct adreno_gpu *adreno_gpu; | |
3031 | struct msm_gpu *gpu; | |
af66706a | 3032 | bool is_a7xx; |
4b565ca5 JC |
3033 | int ret; |
3034 | ||
3035 | a6xx_gpu = kzalloc(sizeof(*a6xx_gpu), GFP_KERNEL); | |
3036 | if (!a6xx_gpu) | |
3037 | return ERR_PTR(-ENOMEM); | |
3038 | ||
3039 | adreno_gpu = &a6xx_gpu->base; | |
3040 | gpu = &adreno_gpu->base; | |
3041 | ||
12abd735 DB |
3042 | mutex_init(&a6xx_gpu->gmu.lock); |
3043 | ||
1707add8 | 3044 | adreno_gpu->registers = NULL; |
4b565ca5 | 3045 | |
5a903a44 KD |
3046 | /* Check if there is a GMU phandle and set it up */ |
3047 | node = of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0); | |
3048 | /* FIXME: How do we gracefully handle this? */ | |
3049 | BUG_ON(!node); | |
3050 | ||
3051 | adreno_gpu->gmu_is_wrapper = of_device_is_compatible(node, "qcom,adreno-gmu-wrapper"); | |
3052 | ||
47bd37f9 RC |
3053 | adreno_gpu->base.hw_apriv = |
3054 | !!(config->info->quirks & ADRENO_QUIRK_HAS_HW_APRIV); | |
604234f3 | 3055 | |
af66706a | 3056 | /* gpu->info only gets assigned in adreno_gpu_init() */ |
1f8c29e8 | 3057 | is_a7xx = config->info->family == ADRENO_7XX_GEN1 || |
d2bcca0c NA |
3058 | config->info->family == ADRENO_7XX_GEN2 || |
3059 | config->info->family == ADRENO_7XX_GEN3; | |
af66706a KD |
3060 | |
3061 | a6xx_llc_slices_init(pdev, a6xx_gpu, is_a7xx); | |
474dadb8 | 3062 | |
47bd37f9 | 3063 | ret = a6xx_set_supported_hw(&pdev->dev, config->info); |
fe7952c6 AO |
3064 | if (ret) { |
3065 | a6xx_destroy(&(a6xx_gpu->base.base)); | |
3066 | return ERR_PTR(ret); | |
3067 | } | |
3068 | ||
af66706a KD |
3069 | if (is_a7xx) |
3070 | ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs_a7xx, 1); | |
3071 | else if (adreno_has_gmu_wrapper(adreno_gpu)) | |
5a903a44 KD |
3072 | ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs_gmuwrapper, 1); |
3073 | else | |
3074 | ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1); | |
4b565ca5 JC |
3075 | if (ret) { |
3076 | a6xx_destroy(&(a6xx_gpu->base.base)); | |
3077 | return ERR_PTR(ret); | |
3078 | } | |
3079 | ||
2c1b7748 RC |
3080 | /* |
3081 | * For now only clamp to idle freq for devices where this is known not | |
3082 | * to cause power supply issues: | |
3083 | */ | |
3084 | if (adreno_is_a618(adreno_gpu) || adreno_is_7c3(adreno_gpu)) | |
6563f60f | 3085 | priv->gpu_clamp_to_idle = true; |
2c1b7748 | 3086 | |
5a903a44 KD |
3087 | if (adreno_has_gmu_wrapper(adreno_gpu)) |
3088 | ret = a6xx_gmu_wrapper_init(a6xx_gpu, node); | |
3089 | else | |
3090 | ret = a6xx_gmu_init(a6xx_gpu, node); | |
c56de483 | 3091 | of_node_put(node); |
4b565ca5 JC |
3092 | if (ret) { |
3093 | a6xx_destroy(&(a6xx_gpu->base.base)); | |
3094 | return ERR_PTR(ret); | |
3095 | } | |
3096 | ||
3097 | if (gpu->aspace) | |
3098 | msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, | |
3099 | a6xx_fault_handler); | |
3100 | ||
8814455a CA |
3101 | a6xx_calc_ubwc_config(adreno_gpu); |
3102 | ||
4b565ca5 JC |
3103 | return gpu; |
3104 | } |