projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'selinux-pr-20220523' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-block.git] / drivers / gpu / drm / msm / adreno / a6xx_gpu.c
CommitLineData
4b565ca5 1// SPDX-License-Identifier: GPL-2.0
e812744c 2/* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */
4b565ca5
JC		 3
4
5#include "msm_gem.h"
6#include "msm_mmu.h"
4241db42 7#include "msm_gpu_trace.h"
4b565ca5
JC		 8#include "a6xx_gpu.h"
9#include "a6xx_gmu.xml.h"
10
474dadb8 11#include <linux/bitfield.h>
a2c3c0a5 12#include <linux/devfreq.h>
474dadb8 13#include <linux/soc/qcom/llcc-qcom.h>
a2c3c0a5 14
abccb9fe
JC		 15#define GPU_PAS_ID 13
16
4b565ca5
JC		 17static inline bool _a6xx_check_idle(struct msm_gpu *gpu)
18{
19 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
20 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
21
22 /* Check that the GMU is idle */
23 if (!a6xx_gmu_isidle(&a6xx_gpu->gmu))
24 return false;
25
26 /* Check tha the CX master is idle */
27 if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) &
28 ~A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER)
29 return false;
30
31 return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) &
32 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT);
33}
34
991a2719 35static bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
4b565ca5
JC		 36{
37 /* wait for CP to drain ringbuffer: */
38 if (!adreno_idle(gpu, ring))
39 return false;
40
41 if (spin_until(_a6xx_check_idle(gpu))) {
42 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
43 gpu->name, __builtin_return_address(0),
44 gpu_read(gpu, REG_A6XX_RBBM_STATUS),
45 gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS),
46 gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
47 gpu_read(gpu, REG_A6XX_CP_RB_WPTR));
48 return false;
49 }
50
51 return true;
52}
53
0710a740 54static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
4b565ca5 55{
d3a569fc
JC		 56 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
57 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
4b565ca5 58
d3a569fc
JC		 59 /* Expanded APRIV doesn't need to issue the WHERE_AM_I opcode */
60 if (a6xx_gpu->has_whereami && !adreno_gpu->base.hw_apriv) {
d3a569fc
JC		 61 OUT_PKT7(ring, CP_WHERE_AM_I, 2);
62 OUT_RING(ring, lower_32_bits(shadowptr(a6xx_gpu, ring)));
63 OUT_RING(ring, upper_32_bits(shadowptr(a6xx_gpu, ring)));
64 }
0710a740
RC		 65}
66
67static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
68{
69 uint32_t wptr;
70 unsigned long flags;
71
72 update_shadow_rptr(gpu, ring);
d3a569fc 73
77c40603 74 spin_lock_irqsave(&ring->preempt_lock, flags);
4b565ca5
JC		 75
76 /* Copy the shadow to the actual register */
77 ring->cur = ring->next;
78
79 /* Make sure to wrap wptr if we need to */
80 wptr = get_wptr(ring);
81
77c40603 82 spin_unlock_irqrestore(&ring->preempt_lock, flags);
4b565ca5
JC		 83
84 /* Make sure everything is posted before making a decision */
85 mb();
86
87 gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);
88}
89
56869210
JC		 90static void get_stats_counter(struct msm_ringbuffer *ring, u32 counter,
91 u64 iova)
92{
93 OUT_PKT7(ring, CP_REG_TO_MEM, 3);
b5e02e11
RC		 94 OUT_RING(ring, CP_REG_TO_MEM_0_REG(counter) |
95 CP_REG_TO_MEM_0_CNT(2) |
96 CP_REG_TO_MEM_0_64B);
56869210
JC		 97 OUT_RING(ring, lower_32_bits(iova));
98 OUT_RING(ring, upper_32_bits(iova));
99}
100
84c31ee1
JC		 101static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu,
102 struct msm_ringbuffer *ring, struct msm_file_private *ctx)
103{
5f9ffe89 104 bool sysprof = refcount_read(&a6xx_gpu->base.base.sysprof_active) > 1;
84c31ee1
JC		 105 phys_addr_t ttbr;
106 u32 asid;
107 u64 memptr = rbmemptr(ring, ttbr0);
108
1d054c9b 109 if (ctx->seqno == a6xx_gpu->base.base.cur_ctx_seqno)
84c31ee1
JC		 110 return;
111
112 if (msm_iommu_pagetable_params(ctx->aspace->mmu, &ttbr, &asid))
113 return;
114
5f9ffe89
RC		 115 if (!sysprof) {
116 /* Turn off protected mode to write to special registers */
117 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
118 OUT_RING(ring, 0);
119
120 OUT_PKT4(ring, REG_A6XX_RBBM_PERFCTR_SRAM_INIT_CMD, 1);
121 OUT_RING(ring, 1);
122 }
123
84c31ee1
JC		 124 /* Execute the table update */
125 OUT_PKT7(ring, CP_SMMU_TABLE_UPDATE, 4);
126 OUT_RING(ring, CP_SMMU_TABLE_UPDATE_0_TTBR0_LO(lower_32_bits(ttbr)));
127
128 OUT_RING(ring,
129 CP_SMMU_TABLE_UPDATE_1_TTBR0_HI(upper_32_bits(ttbr)) |
130 CP_SMMU_TABLE_UPDATE_1_ASID(asid));
131 OUT_RING(ring, CP_SMMU_TABLE_UPDATE_2_CONTEXTIDR(0));
132 OUT_RING(ring, CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK(0));
133
134 /*
135 * Write the new TTBR0 to the memstore. This is good for debugging.
136 */
137 OUT_PKT7(ring, CP_MEM_WRITE, 4);
138 OUT_RING(ring, CP_MEM_WRITE_0_ADDR_LO(lower_32_bits(memptr)));
139 OUT_RING(ring, CP_MEM_WRITE_1_ADDR_HI(upper_32_bits(memptr)));
140 OUT_RING(ring, lower_32_bits(ttbr));
141 OUT_RING(ring, (asid << 16) | upper_32_bits(ttbr));
142
143 /*
144 * And finally, trigger a uche flush to be sure there isn't anything
145 * lingering in that part of the GPU
146 */
147
148 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
149 OUT_RING(ring, 0x31);
5f9ffe89
RC		 150
151 if (!sysprof) {
152 /*
153 * Wait for SRAM clear after the pgtable update, so the
154 * two can happen in parallel:
155 */
156 OUT_PKT7(ring, CP_WAIT_REG_MEM, 6);
157 OUT_RING(ring, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ));
158 OUT_RING(ring, CP_WAIT_REG_MEM_1_POLL_ADDR_LO(
159 REG_A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS));
160 OUT_RING(ring, CP_WAIT_REG_MEM_2_POLL_ADDR_HI(0));
161 OUT_RING(ring, CP_WAIT_REG_MEM_3_REF(0x1));
162 OUT_RING(ring, CP_WAIT_REG_MEM_4_MASK(0x1));
163 OUT_RING(ring, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(0));
164
165 /* Re-enable protected mode: */
166 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
167 OUT_RING(ring, 1);
168 }
84c31ee1
JC		 169}
170
15eb9ad0 171static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
4b565ca5 172{
56869210 173 unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT;
4241db42
JC		 174 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
175 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
4b565ca5 176 struct msm_ringbuffer *ring = submit->ring;
0710a740 177 unsigned int i, ibs = 0;
4b565ca5 178
84c31ee1
JC		 179 a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx);
180
cc4c26d4 181 get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
56869210
JC		 182 rbmemptr_stats(ring, index, cpcycles_start));
183
184 /*
185 * For PM4 the GMU register offsets are calculated from the base of the
186 * GPU registers so we need to add 0x1a800 to the register value on A630
187 * to get the right value from PM4.
188 */
40843403 189 get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
56869210
JC		 190 rbmemptr_stats(ring, index, alwayson_start));
191
4b565ca5
JC		 192 /* Invalidate CCU depth and color */
193 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
b5e02e11 194 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(PC_CCU_INVALIDATE_DEPTH));
4b565ca5
JC		 195
196 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
b5e02e11 197 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(PC_CCU_INVALIDATE_COLOR));
4b565ca5
JC		 198
199 /* Submit the commands */
200 for (i = 0; i < submit->nr_cmds; i++) {
201 switch (submit->cmd[i].type) {
202 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
203 break;
204 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
1d054c9b 205 if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno)
4b565ca5 206 break;
df561f66 207 fallthrough;
4b565ca5
JC		 208 case MSM_SUBMIT_CMD_BUF:
209 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
210 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
211 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
212 OUT_RING(ring, submit->cmd[i].size);
0710a740 213 ibs++;
4b565ca5
JC		 214 break;
215 }
0710a740
RC		 216
217 /*
218 * Periodically update shadow-wptr if needed, so that we
219 * can see partial progress of submits with large # of
220 * cmds.. otherwise we could needlessly stall waiting for
221 * ringbuffer state, simply due to looking at a shadow
222 * rptr value that has not been updated
223 */
224 if ((ibs % 32) == 0)
225 update_shadow_rptr(gpu, ring);
4b565ca5
JC		 226 }
227
cc4c26d4 228 get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
56869210 229 rbmemptr_stats(ring, index, cpcycles_end));
40843403 230 get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
56869210
JC		 231 rbmemptr_stats(ring, index, alwayson_end));
232
4b565ca5
JC		 233 /* Write the fence to the scratch register */
234 OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1);
235 OUT_RING(ring, submit->seqno);
236
237 /*
238 * Execute a CACHE_FLUSH_TS event. This will ensure that the
239 * timestamp is written to the memory and then triggers the interrupt
240 */
241 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
b5e02e11
RC		 242 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) |
243 CP_EVENT_WRITE_0_IRQ);
4b565ca5
JC		 244 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
245 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
246 OUT_RING(ring, submit->seqno);
247
4241db42 248 trace_msm_gpu_submit_flush(submit,
40843403
JM		 249 gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
250 REG_A6XX_CP_ALWAYS_ON_COUNTER_HI));
4241db42 251
4b565ca5
JC		 252 a6xx_flush(gpu, ring);
253}
254
b1c53a2a 255const struct adreno_reglist a630_hwcg[] = {
4b565ca5
JC		 256 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222},
257 {REG_A6XX_RBBM_CLOCK_CNTL_SP1, 0x22222222},
258 {REG_A6XX_RBBM_CLOCK_CNTL_SP2, 0x22222222},
259 {REG_A6XX_RBBM_CLOCK_CNTL_SP3, 0x22222222},
260 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220},
261 {REG_A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220},
262 {REG_A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220},
263 {REG_A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220},
264 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
265 {REG_A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
266 {REG_A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
267 {REG_A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
268 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf},
269 {REG_A6XX_RBBM_CLOCK_HYST_SP1, 0x0000f3cf},
270 {REG_A6XX_RBBM_CLOCK_HYST_SP2, 0x0000f3cf},
271 {REG_A6XX_RBBM_CLOCK_HYST_SP3, 0x0000f3cf},
272 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
273 {REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222},
274 {REG_A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222},
275 {REG_A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222},
276 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
277 {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
278 {REG_A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
279 {REG_A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
280 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
281 {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222},
282 {REG_A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222},
283 {REG_A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222},
284 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
285 {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222},
286 {REG_A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222},
287 {REG_A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222},
288 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
289 {REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
290 {REG_A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
291 {REG_A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
292 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
293 {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
294 {REG_A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
295 {REG_A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
296 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
297 {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777},
298 {REG_A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777},
299 {REG_A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777},
300 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
301 {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777},
302 {REG_A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777},
303 {REG_A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777},
304 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
305 {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
306 {REG_A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
307 {REG_A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
308 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
309 {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
310 {REG_A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
311 {REG_A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
312 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
313 {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111},
314 {REG_A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111},
315 {REG_A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111},
316 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
317 {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111},
318 {REG_A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111},
319 {REG_A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111},
320 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
321 {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
322 {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
323 {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
324 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
325 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
326 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
327 {REG_A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
328 {REG_A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
329 {REG_A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
330 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222},
331 {REG_A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222},
332 {REG_A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222},
333 {REG_A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222},
334 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
335 {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220},
336 {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220},
337 {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220},
338 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00},
339 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040f00},
340 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040f00},
341 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040f00},
342 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022},
343 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
344 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
345 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
346 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
347 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
348 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
349 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
350 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
351 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
352 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
353 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
354 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
355 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
356 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
357 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
358 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
359 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
b1c53a2a
JM		 360 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
361 {},
4b565ca5
JC		 362};
363
66ffb915
JM		 364const struct adreno_reglist a640_hwcg[] = {
365 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
366 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
367 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
368 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
369 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
370 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
371 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
372 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
373 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
374 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
375 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
376 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
377 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
378 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
379 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
380 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
381 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
382 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
383 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
384 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
385 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05222022},
386 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
387 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
388 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
389 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
390 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
391 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
392 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
393 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
394 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
395 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
396 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
397 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
398 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
399 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
400 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
401 {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
402 {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
403 {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
404 {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
405 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
406 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
407 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
408 {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
409 {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
410 {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
411 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
412 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
413 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
414 {},
415};
416
417const struct adreno_reglist a650_hwcg[] = {
418 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
419 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
420 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
421 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
422 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
423 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
424 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
425 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
426 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
427 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
428 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
429 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
430 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
431 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
432 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
433 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
434 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
435 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
436 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
437 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
438 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
439 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
440 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
441 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
442 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
443 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
444 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
445 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
446 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
447 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
448 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
449 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
450 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
451 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
452 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
453 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
454 {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
455 {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
456 {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
457 {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000777},
458 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
459 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
460 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
461 {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
462 {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
463 {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
464 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
465 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
466 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
467 {},
468};
469
f6d62d09
JM		 470const struct adreno_reglist a660_hwcg[] = {
471 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
472 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
473 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
474 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
475 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
476 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
477 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
478 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
479 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
480 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
481 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
482 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
483 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
484 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
485 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
486 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
487 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
488 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
489 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
490 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
491 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
492 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
493 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
494 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
495 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
496 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
497 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
498 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
499 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
500 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
501 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
502 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
503 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
504 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
505 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
506 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
507 {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
508 {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
509 {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
510 {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
511 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
512 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
513 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
514 {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
515 {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
516 {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
517 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
518 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
519 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
520 {},
521};
522
4b565ca5
JC		 523static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)
524{
525 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
526 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
527 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
b1c53a2a 528 const struct adreno_reglist *reg;
4b565ca5 529 unsigned int i;
66ffb915 530 u32 val, clock_cntl_on;
4b565ca5 531
b1c53a2a
JM		 532 if (!adreno_gpu->info->hwcg)
533 return;
534
66ffb915
JM		 535 if (adreno_is_a630(adreno_gpu))
536 clock_cntl_on = 0x8aa8aa02;
537 else
538 clock_cntl_on = 0x8aa8aa82;
539
4b565ca5
JC		 540 val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL);
541
542 /* Don't re-program the registers if they are already correct */
66ffb915 543 if ((!state && !val) || (state && (val == clock_cntl_on)))
4b565ca5
JC		 544 return;
545
546 /* Disable SP clock before programming HWCG registers */
547 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0);
548
b1c53a2a
JM		 549 for (i = 0; (reg = &adreno_gpu->info->hwcg[i], reg->offset); i++)
550 gpu_write(gpu, reg->offset, state ? reg->value : 0);
4b565ca5
JC		 551
552 /* Enable SP clock */
553 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1);
554
66ffb915 555 gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? clock_cntl_on : 0);
4b565ca5
JC		 556}
557
40843403
JM		 558/* For a615, a616, a618, A619, a630, a640 and a680 */
559static const u32 a6xx_protect[] = {
560 A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
561 A6XX_PROTECT_RDONLY(0x00501, 0x0005),
562 A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
563 A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
564 A6XX_PROTECT_NORDWR(0x00510, 0x0000),
565 A6XX_PROTECT_NORDWR(0x00534, 0x0000),
566 A6XX_PROTECT_NORDWR(0x00800, 0x0082),
567 A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
568 A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
569 A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
570 A6XX_PROTECT_NORDWR(0x00900, 0x004d),
571 A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
572 A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
573 A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
574 A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
575 A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
576 A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
577 A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
578 A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
579 A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
580 A6XX_PROTECT_NORDWR(0x09624, 0x01db),
581 A6XX_PROTECT_NORDWR(0x09e70, 0x0001),
582 A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
583 A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
584 A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
585 A6XX_PROTECT_NORDWR(0x0ae50, 0x032f),
586 A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
587 A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
588 A6XX_PROTECT_NORDWR(0x0be20, 0x17df),
589 A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
590 A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
591 A6XX_PROTECT_NORDWR(0x11c00, 0x0000), /* note: infinite range */
592};
593
594/* These are for a620 and a650 */
595static const u32 a650_protect[] = {
596 A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
597 A6XX_PROTECT_RDONLY(0x00501, 0x0005),
598 A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
599 A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
600 A6XX_PROTECT_NORDWR(0x00510, 0x0000),
601 A6XX_PROTECT_NORDWR(0x00534, 0x0000),
602 A6XX_PROTECT_NORDWR(0x00800, 0x0082),
603 A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
604 A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
605 A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
606 A6XX_PROTECT_NORDWR(0x00900, 0x004d),
607 A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
608 A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
609 A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
610 A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
611 A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
612 A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
613 A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
614 A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
615 A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
616 A6XX_PROTECT_NORDWR(0x08e80, 0x027f),
617 A6XX_PROTECT_NORDWR(0x09624, 0x01db),
618 A6XX_PROTECT_NORDWR(0x09e60, 0x0011),
619 A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
620 A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
621 A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
622 A6XX_PROTECT_NORDWR(0x0ae50, 0x032f),
623 A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
624 A6XX_PROTECT_NORDWR(0x0b608, 0x0007),
625 A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
626 A6XX_PROTECT_NORDWR(0x0be20, 0x17df),
627 A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
628 A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
629 A6XX_PROTECT_NORDWR(0x18400, 0x1fff),
630 A6XX_PROTECT_NORDWR(0x1a800, 0x1fff),
631 A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
632 A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
633 A6XX_PROTECT_NORDWR(0x1f887, 0x001b),
634 A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000), /* note: infinite range */
635};
636
f6d62d09
JM		 637/* These are for a635 and a660 */
638static const u32 a660_protect[] = {
639 A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
640 A6XX_PROTECT_RDONLY(0x00501, 0x0005),
641 A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
642 A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
643 A6XX_PROTECT_NORDWR(0x00510, 0x0000),
644 A6XX_PROTECT_NORDWR(0x00534, 0x0000),
645 A6XX_PROTECT_NORDWR(0x00800, 0x0082),
646 A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
647 A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
648 A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
649 A6XX_PROTECT_NORDWR(0x00900, 0x004d),
650 A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
651 A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
652 A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
653 A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
654 A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
655 A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
656 A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
657 A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
658 A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
659 A6XX_PROTECT_NORDWR(0x08e80, 0x027f),
660 A6XX_PROTECT_NORDWR(0x09624, 0x01db),
661 A6XX_PROTECT_NORDWR(0x09e60, 0x0011),
662 A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
663 A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
664 A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
665 A6XX_PROTECT_NORDWR(0x0ae50, 0x012f),
666 A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
667 A6XX_PROTECT_NORDWR(0x0b608, 0x0006),
668 A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
669 A6XX_PROTECT_NORDWR(0x0be20, 0x015f),
670 A6XX_PROTECT_NORDWR(0x0d000, 0x05ff),
671 A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
672 A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
673 A6XX_PROTECT_NORDWR(0x18400, 0x1fff),
674 A6XX_PROTECT_NORDWR(0x1a400, 0x1fff),
675 A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
676 A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
677 A6XX_PROTECT_NORDWR(0x1f860, 0x0000),
678 A6XX_PROTECT_NORDWR(0x1f887, 0x001b),
679 A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000), /* note: infinite range */
680};
681
40843403
JM		 682static void a6xx_set_cp_protect(struct msm_gpu *gpu)
683{
684 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
685 const u32 *regs = a6xx_protect;
cca96584 686 unsigned i, count, count_max;
40843403
JM		 687
688 if (adreno_is_a650(adreno_gpu)) {
689 regs = a650_protect;
690 count = ARRAY_SIZE(a650_protect);
691 count_max = 48;
cca96584 692 BUILD_BUG_ON(ARRAY_SIZE(a650_protect) > 48);
192f4ee3 693 } else if (adreno_is_a660_family(adreno_gpu)) {
f6d62d09
JM		 694 regs = a660_protect;
695 count = ARRAY_SIZE(a660_protect);
696 count_max = 48;
cca96584
RC		 697 BUILD_BUG_ON(ARRAY_SIZE(a660_protect) > 48);
698 } else {
699 regs = a6xx_protect;
700 count = ARRAY_SIZE(a6xx_protect);
701 count_max = 32;
702 BUILD_BUG_ON(ARRAY_SIZE(a6xx_protect) > 32);
40843403
JM		 703 }
704
705 /*
706 * Enable access protection to privileged registers, fault on an access
707 * protect violation and select the last span to protect from the start
708 * address all the way to the end of the register address space
709 */
710 gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, BIT(0) | BIT(1) | BIT(3));
711
712 for (i = 0; i < count - 1; i++)
713 gpu_write(gpu, REG_A6XX_CP_PROTECT(i), regs[i]);
714 /* last CP_PROTECT to have "infinite" length on the last entry */
715 gpu_write(gpu, REG_A6XX_CP_PROTECT(count_max - 1), regs[i]);
716}
717
d0bac4e9
JM		 718static void a6xx_set_ubwc_config(struct msm_gpu *gpu)
719{
720 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
721 u32 lower_bit = 2;
722 u32 amsbc = 0;
723 u32 rgb565_predicator = 0;
724 u32 uavflagprd_inv = 0;
725
726 /* a618 is using the hw default values */
727 if (adreno_is_a618(adreno_gpu))
728 return;
729
083cc3a4 730 if (adreno_is_a640_family(adreno_gpu))
d0bac4e9
JM		 731 amsbc = 1;
732
f6d62d09 733 if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu)) {
d0bac4e9
JM		 734 /* TODO: get ddr type from bootloader and use 2 for LPDDR4 */
735 lower_bit = 3;
736 amsbc = 1;
737 rgb565_predicator = 1;
738 uavflagprd_inv = 2;
739 }
740
192f4ee3
AO		 741 if (adreno_is_7c3(adreno_gpu)) {
742 lower_bit = 1;
743 amsbc = 1;
744 rgb565_predicator = 1;
745 uavflagprd_inv = 2;
746 }
747
d0bac4e9
JM		 748 gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL,
749 rgb565_predicator << 11 | amsbc << 4 | lower_bit << 1);
750 gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, lower_bit << 1);
751 gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL,
b4387eaf 752 uavflagprd_inv << 4 | lower_bit << 1);
d0bac4e9
JM		 753 gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL, lower_bit << 21);
754}
755
4b565ca5
JC		 756static int a6xx_cp_init(struct msm_gpu *gpu)
757{
758 struct msm_ringbuffer *ring = gpu->rb[0];
759
760 OUT_PKT7(ring, CP_ME_INIT, 8);
761
762 OUT_RING(ring, 0x0000002f);
763
764 /* Enable multiple hardware contexts */
765 OUT_RING(ring, 0x00000003);
766
767 /* Enable error detection */
768 OUT_RING(ring, 0x20000000);
769
770 /* Don't enable header dump */
771 OUT_RING(ring, 0x00000000);
772 OUT_RING(ring, 0x00000000);
773
774 /* No workarounds enabled */
775 OUT_RING(ring, 0x00000000);
776
777 /* Pad rest of the cmds with 0's */
778 OUT_RING(ring, 0x00000000);
779 OUT_RING(ring, 0x00000000);
780
781 a6xx_flush(gpu, ring);
782 return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
783}
784
8490f02a
JC		 785/*
786 * Check that the microcode version is new enough to include several key
787 * security fixes. Return true if the ucode is safe.
788 */
789static bool a6xx_ucode_check_version(struct a6xx_gpu *a6xx_gpu,
d3a569fc
JC		 790 struct drm_gem_object *obj)
791{
8490f02a
JC		 792 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
793 struct msm_gpu *gpu = &adreno_gpu->base;
f3a6b02c 794 const char *sqe_name = adreno_gpu->info->fw[ADRENO_FW_SQE];
96c876f1 795 u32 *buf = msm_gem_get_vaddr(obj);
8490f02a 796 bool ret = false;
d3a569fc
JC		 797
798 if (IS_ERR(buf))
8490f02a 799 return false;
d3a569fc
JC		 800
801 /*
8490f02a
JC		 802 * Targets up to a640 (a618, a630 and a640) need to check for a
803 * microcode version that is patched to support the whereami opcode or
804 * one that is new enough to include it by default.
f6d62d09
JM		 805 *
806 * a650 tier targets don't need whereami but still need to be
807 * equal to or newer than 0.95 for other security fixes
808 *
809 * a660 targets have all the critical security fixes from the start
d3a569fc 810 */
f3a6b02c 811 if (!strcmp(sqe_name, "a630_sqe.fw")) {
8490f02a
JC		 812 /*
813 * If the lowest nibble is 0xa that is an indication that this
814 * microcode has been patched. The actual version is in dword
815 * [3] but we only care about the patchlevel which is the lowest
816 * nibble of dword [3]
817 *
818 * Otherwise check that the firmware is greater than or equal
819 * to 1.90 which was the first version that had this fix built
820 * in
821 */
822 if ((((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1) ||
823 (buf[0] & 0xfff) >= 0x190) {
824 a6xx_gpu->has_whereami = true;
825 ret = true;
826 goto out;
827 }
828
829 DRM_DEV_ERROR(&gpu->pdev->dev,
830 "a630 SQE ucode is too old. Have version %x need at least %x\n",
831 buf[0] & 0xfff, 0x190);
f3a6b02c 832 } else if (!strcmp(sqe_name, "a650_sqe.fw")) {
f6d62d09
JM		 833 if ((buf[0] & 0xfff) >= 0x095) {
834 ret = true;
835 goto out;
8490f02a 836 }
d3a569fc 837
f6d62d09
JM		 838 DRM_DEV_ERROR(&gpu->pdev->dev,
839 "a650 SQE ucode is too old. Have version %x need at least %x\n",
840 buf[0] & 0xfff, 0x095);
f3a6b02c 841 } else if (!strcmp(sqe_name, "a660_sqe.fw")) {
f6d62d09
JM		 842 ret = true;
843 } else {
844 DRM_DEV_ERROR(&gpu->pdev->dev,
845 "unknown GPU, add it to a6xx_ucode_check_version()!!\n");
8490f02a
JC		 846 }
847out:
d3a569fc 848 msm_gem_put_vaddr(obj);
8490f02a 849 return ret;
d3a569fc
JC		 850}
851
4b565ca5
JC		 852static int a6xx_ucode_init(struct msm_gpu *gpu)
853{
854 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
855 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
856
857 if (!a6xx_gpu->sqe_bo) {
858 a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu,
859 adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova);
860
861 if (IS_ERR(a6xx_gpu->sqe_bo)) {
862 int ret = PTR_ERR(a6xx_gpu->sqe_bo);
863
864 a6xx_gpu->sqe_bo = NULL;
865 DRM_DEV_ERROR(&gpu->pdev->dev,
866 "Could not allocate SQE ucode: %d\n", ret);
867
868 return ret;
869 }
0815d774
JC		 870
871 msm_gem_object_set_name(a6xx_gpu->sqe_bo, "sqefw");
8490f02a
JC		 872 if (!a6xx_ucode_check_version(a6xx_gpu, a6xx_gpu->sqe_bo)) {
873 msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);
874 drm_gem_object_put(a6xx_gpu->sqe_bo);
875
876 a6xx_gpu->sqe_bo = NULL;
877 return -EPERM;
878 }
4b565ca5
JC		 879 }
880
cc4c26d4
RC		 881 gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE,
882 REG_A6XX_CP_SQE_INSTR_BASE+1, a6xx_gpu->sqe_iova);
4b565ca5
JC		 883
884 return 0;
885}
886
abccb9fe
JC		 887static int a6xx_zap_shader_init(struct msm_gpu *gpu)
888{
889 static bool loaded;
890 int ret;
891
892 if (loaded)
893 return 0;
894
895 ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);
896
897 loaded = !ret;
898 return ret;
899}
900
4b565ca5
JC		 901#define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
902 A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
903 A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
904 A6XX_RBBM_INT_0_MASK_CP_IB2 | \
905 A6XX_RBBM_INT_0_MASK_CP_IB1 | \
906 A6XX_RBBM_INT_0_MASK_CP_RB | \
907 A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
908 A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
909 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
910 A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
911 A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
912
f6f59072 913static int hw_init(struct msm_gpu *gpu)
4b565ca5
JC		 914{
915 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
916 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
917 int ret;
918
919 /* Make sure the GMU keeps the GPU on while we set it up */
920 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
921
922 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);
923
924 /*
925 * Disable the trusted memory range - we don't actually supported secure
926 * memory rendering at this point in time and we don't want to block off
927 * part of the virtual memory space.
928 */
929 gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
930 REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
931 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
932
adf151c2
JC		 933 /* Turn on 64 bit addressing for all blocks */
934 gpu_write(gpu, REG_A6XX_CP_ADDR_MODE_CNTL, 0x1);
935 gpu_write(gpu, REG_A6XX_VSC_ADDR_MODE_CNTL, 0x1);
936 gpu_write(gpu, REG_A6XX_GRAS_ADDR_MODE_CNTL, 0x1);
937 gpu_write(gpu, REG_A6XX_RB_ADDR_MODE_CNTL, 0x1);
938 gpu_write(gpu, REG_A6XX_PC_ADDR_MODE_CNTL, 0x1);
939 gpu_write(gpu, REG_A6XX_HLSQ_ADDR_MODE_CNTL, 0x1);
940 gpu_write(gpu, REG_A6XX_VFD_ADDR_MODE_CNTL, 0x1);
941 gpu_write(gpu, REG_A6XX_VPC_ADDR_MODE_CNTL, 0x1);
942 gpu_write(gpu, REG_A6XX_UCHE_ADDR_MODE_CNTL, 0x1);
943 gpu_write(gpu, REG_A6XX_SP_ADDR_MODE_CNTL, 0x1);
944 gpu_write(gpu, REG_A6XX_TPL1_ADDR_MODE_CNTL, 0x1);
945 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);
946
b1c53a2a
JM		 947 /* enable hardware clockgating */
948 a6xx_set_hwcg(gpu, true);
4b565ca5 949
e812744c 950 /* VBIF/GBIF start*/
083cc3a4
RC		 951 if (adreno_is_a640_family(adreno_gpu) ||
952 adreno_is_a650_family(adreno_gpu)) {
24e6938e
JM		 953 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE0, 0x00071620);
954 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE1, 0x00071620);
955 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE2, 0x00071620);
956 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620);
957 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620);
958 gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x3);
959 } else {
960 gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);
961 }
962
e812744c
SM		 963 if (adreno_is_a630(adreno_gpu))
964 gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
4b565ca5
JC		 965
966 /* Make all blocks contribute to the GPU BUSY perf counter */
967 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);
968
969 /* Disable L2 bypass in the UCHE */
970 gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0);
971 gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff);
972 gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_LO, 0xfffff000);
973 gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff);
974 gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000);
975 gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff);
976
f6d62d09 977 if (!adreno_is_a650_family(adreno_gpu)) {
24e6938e
JM		 978 /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */
979 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO,
980 REG_A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x00100000);
4b565ca5 981
24e6938e
JM		 982 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO,
983 REG_A6XX_UCHE_GMEM_RANGE_MAX_HI,
984 0x00100000 + adreno_gpu->gmem - 1);
985 }
4b565ca5
JC		 986
987 gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804);
988 gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4);
989
083cc3a4
RC		 990 if (adreno_is_a640_family(adreno_gpu) ||
991 adreno_is_a650_family(adreno_gpu))
24e6938e
JM		 992 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140);
993 else
994 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);
4b565ca5
JC		 995 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
996
192f4ee3 997 if (adreno_is_a660_family(adreno_gpu))
f6d62d09
JM		 998 gpu_write(gpu, REG_A6XX_CP_LPAC_PROG_FIFO_SIZE, 0x00000020);
999
4b565ca5
JC		 1000 /* Setting the mem pool size */
1001 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128);
1002
564499f5
JM		 1003 /* Setting the primFifo thresholds default values,
1004 * and vccCacheSkipDis=1 bit (0x200) for A640 and newer
1005 */
840d10b6
BA		 1006 if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu))
1007 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200);
083cc3a4 1008 else if (adreno_is_a640_family(adreno_gpu) || adreno_is_7c3(adreno_gpu))
564499f5 1009 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00200200);
192f4ee3
AO		 1010 else if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu))
1011 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200);
24e6938e 1012 else
564499f5 1013 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00180000);
4b565ca5
JC		 1014
1015 /* Set the AHB default slave response to "ERROR" */
1016 gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1);
1017
1018 /* Turn on performance counters */
1019 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1);
1020
1021 /* Select CP0 to always count cycles */
cc4c26d4 1022 gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL(0), PERF_CP_ALWAYS_COUNT);
4b565ca5 1023
d0bac4e9 1024 a6xx_set_ubwc_config(gpu);
4b565ca5
JC		 1025
1026 /* Enable fault detection */
1027 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL,
1028 (1 << 30) | 0x1fffff);
1029
1030 gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, 1);
1031
24e6938e 1032 /* Set weights for bicubic filtering */
f6d62d09 1033 if (adreno_is_a650_family(adreno_gpu)) {
24e6938e
JM		 1034 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0, 0);
1035 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1,
1036 0x3fe05ff4);
1037 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2,
1038 0x3fa0ebee);
1039 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3,
1040 0x3f5193ed);
1041 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4,
1042 0x3f0243f0);
1043 }
1044
4b565ca5 1045 /* Protect registers from the CP */
40843403 1046 a6xx_set_cp_protect(gpu);
4b565ca5 1047
192f4ee3 1048 if (adreno_is_a660_family(adreno_gpu)) {
f6d62d09
JM		 1049 gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, 0x1);
1050 gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x0);
f6d62d09
JM		 1051 }
1052
192f4ee3
AO		 1053 /* Set dualQ + disable afull for A660 GPU */
1054 if (adreno_is_a660(adreno_gpu))
1055 gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x66906);
1056
604234f3
JC		 1057 /* Enable expanded apriv for targets that support it */
1058 if (gpu->hw_apriv) {
24e6938e
JM		 1059 gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL,
1060 (1 << 6) | (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1));
1061 }
1062
4b565ca5
JC		 1063 /* Enable interrupts */
1064 gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_INT_MASK);
1065
1066 ret = adreno_hw_init(gpu);
1067 if (ret)
1068 goto out;
1069
1070 ret = a6xx_ucode_init(gpu);
1071 if (ret)
1072 goto out;
1073
f6828e0c
JC		 1074 /* Set the ringbuffer address */
1075 gpu_write64(gpu, REG_A6XX_CP_RB_BASE, REG_A6XX_CP_RB_BASE_HI,
1076 gpu->rb[0]->iova);
1077
d3a569fc
JC		 1078 /* Targets that support extended APRIV can use the RPTR shadow from
1079 * hardware but all the other ones need to disable the feature. Targets
1080 * that support the WHERE_AM_I opcode can use that instead
1081 */
1082 if (adreno_gpu->base.hw_apriv)
1083 gpu_write(gpu, REG_A6XX_CP_RB_CNTL, MSM_GPU_RB_CNTL_DEFAULT);
1084 else
1085 gpu_write(gpu, REG_A6XX_CP_RB_CNTL,
1086 MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
1087
1088 /*
1089 * Expanded APRIV and targets that support WHERE_AM_I both need a
1090 * privileged buffer to store the RPTR shadow
1091 */
1092
1093 if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) {
1094 if (!a6xx_gpu->shadow_bo) {
030af2b0 1095 a6xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
d3a569fc 1096 sizeof(u32) * gpu->nr_rings,
a5fc7aa9 1097 MSM_BO_WC | MSM_BO_MAP_PRIV,
d3a569fc
JC		 1098 gpu->aspace, &a6xx_gpu->shadow_bo,
1099 &a6xx_gpu->shadow_iova);
1100
1101 if (IS_ERR(a6xx_gpu->shadow))
1102 return PTR_ERR(a6xx_gpu->shadow);
8b9af498
RC		 1103
1104 msm_gem_object_set_name(a6xx_gpu->shadow_bo, "shadow");
d3a569fc
JC		 1105 }
1106
1107 gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR_LO,
1108 REG_A6XX_CP_RB_RPTR_ADDR_HI,
1109 shadowptr(a6xx_gpu, gpu->rb[0]));
1110 }
f6828e0c 1111
4b565ca5
JC		 1112 /* Always come up on rb 0 */
1113 a6xx_gpu->cur_ring = gpu->rb[0];
1114
1d054c9b 1115 gpu->cur_ctx_seqno = 0;
84c31ee1 1116
4b565ca5
JC		 1117 /* Enable the SQE_to start the CP engine */
1118 gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1);
1119
1120 ret = a6xx_cp_init(gpu);
1121 if (ret)
1122 goto out;
1123
abccb9fe
JC		 1124 /*
1125 * Try to load a zap shader into the secure world. If successful
1126 * we can use the CP to switch out of secure mode. If not then we
1127 * have no resource but to try to switch ourselves out manually. If we
1128 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
1129 * be blocked and a permissions violation will soon follow.
1130 */
1131 ret = a6xx_zap_shader_init(gpu);
1132 if (!ret) {
1133 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);
1134 OUT_RING(gpu->rb[0], 0x00000000);
1135
1136 a6xx_flush(gpu, gpu->rb[0]);
1137 if (!a6xx_idle(gpu, gpu->rb[0]))
1138 return -EINVAL;
15ab987c
RC		 1139 } else if (ret == -ENODEV) {
1140 /*
1141 * This device does not use zap shader (but print a warning
1142 * just in case someone got their dt wrong.. hopefully they
1143 * have a debug UART to realize the error of their ways...
1144 * if you mess this up you are about to crash horribly)
1145 */
abccb9fe
JC		 1146 dev_warn_once(gpu->dev->dev,
1147 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
1148 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
15273ffd 1149 ret = 0;
15ab987c
RC		 1150 } else {
1151 return ret;
abccb9fe 1152 }
4b565ca5
JC		 1153
1154out:
1155 /*
1156 * Tell the GMU that we are done touching the GPU and it can start power
1157 * management
1158 */
1159 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
1160
8167e6fa
JM		 1161 if (a6xx_gpu->gmu.legacy) {
1162 /* Take the GMU out of its special boot mode */
1163 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER);
1164 }
4b565ca5
JC		 1165
1166 return ret;
1167}
1168
f6f59072
RC		 1169static int a6xx_hw_init(struct msm_gpu *gpu)
1170{
1171 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1172 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1173 int ret;
1174
1175 mutex_lock(&a6xx_gpu->gmu.lock);
1176 ret = hw_init(gpu);
1177 mutex_unlock(&a6xx_gpu->gmu.lock);
1178
1179 return ret;
1180}
1181
4b565ca5
JC		 1182static void a6xx_dump(struct msm_gpu *gpu)
1183{
6a41da17 1184 DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n",
4b565ca5
JC		 1185 gpu_read(gpu, REG_A6XX_RBBM_STATUS));
1186 adreno_dump(gpu);
1187}
1188
1189#define VBIF_RESET_ACK_TIMEOUT 100
1190#define VBIF_RESET_ACK_MASK 0x00f0
1191
1192static void a6xx_recover(struct msm_gpu *gpu)
1193{
1194 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1195 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1196 int i;
1197
1198 adreno_dump_info(gpu);
1199
1200 for (i = 0; i < 8; i++)
6a41da17 1201 DRM_DEV_INFO(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i,
4b565ca5
JC		 1202 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i)));
1203
1204 if (hang_debug)
1205 a6xx_dump(gpu);
1206
1207 /*
1208 * Turn off keep alive that might have been enabled by the hang
1209 * interrupt
1210 */
1211 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0);
1212
1213 gpu->funcs->pm_suspend(gpu);
1214 gpu->funcs->pm_resume(gpu);
1215
1216 msm_gpu_hw_init(gpu);
1217}
1218
2a574cc0
JC		 1219static const char *a6xx_uche_fault_block(struct msm_gpu *gpu, u32 mid)
1220{
1221 static const char *uche_clients[7] = {
1222 "VFD", "SP", "VSC", "VPC", "HLSQ", "PC", "LRZ",
1223 };
1224 u32 val;
1225
1226 if (mid < 1 || mid > 3)
1227 return "UNKNOWN";
1228
1229 /*
1230 * The source of the data depends on the mid ID read from FSYNR1.
1231 * and the client ID read from the UCHE block
1232 */
1233 val = gpu_read(gpu, REG_A6XX_UCHE_CLIENT_PF);
1234
1235 /* mid = 3 is most precise and refers to only one block per client */
1236 if (mid == 3)
1237 return uche_clients[val & 7];
1238
1239 /* For mid=2 the source is TP or VFD except when the client id is 0 */
1240 if (mid == 2)
1241 return ((val & 7) == 0) ? "TP" : "TP|VFD";
1242
1243 /* For mid=1 just return "UCHE" as a catchall for everything else */
1244 return "UCHE";
1245}
1246
1247static const char *a6xx_fault_block(struct msm_gpu *gpu, u32 id)
1248{
1249 if (id == 0)
1250 return "CP";
1251 else if (id == 4)
1252 return "CCU";
1253 else if (id == 6)
1254 return "CDP Prefetch";
1255
1256 return a6xx_uche_fault_block(gpu, id);
1257}
1258
1259#define ARM_SMMU_FSR_TF BIT(1)
1260#define ARM_SMMU_FSR_PF BIT(3)
1261#define ARM_SMMU_FSR_EF BIT(4)
1262
1263static int a6xx_fault_handler(void *arg, unsigned long iova, int flags, void *data)
4b565ca5
JC		 1264{
1265 struct msm_gpu *gpu = arg;
2a574cc0
JC		 1266 struct adreno_smmu_fault_info *info = data;
1267 const char *type = "UNKNOWN";
e25e92e0
RC		 1268 const char *block;
1269 bool do_devcoredump = info && !READ_ONCE(gpu->crashstate);
1270
1271 /*
1272 * If we aren't going to be resuming later from fault_worker, then do
1273 * it now.
1274 */
1275 if (!do_devcoredump) {
1276 gpu->aspace->mmu->funcs->resume_translation(gpu->aspace->mmu);
1277 }
4b565ca5 1278
2a574cc0
JC		 1279 /*
1280 * Print a default message if we couldn't get the data from the
1281 * adreno-smmu-priv
1282 */
1283 if (!info) {
1284 pr_warn_ratelimited("*** gpu fault: iova=%.16lx flags=%d (%u,%u,%u,%u)\n",
4b565ca5
JC		 1285 iova, flags,
1286 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),
1287 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),
1288 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),
1289 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)));
1290
2a574cc0
JC		 1291 return 0;
1292 }
1293
1294 if (info->fsr & ARM_SMMU_FSR_TF)
1295 type = "TRANSLATION";
1296 else if (info->fsr & ARM_SMMU_FSR_PF)
1297 type = "PERMISSION";
1298 else if (info->fsr & ARM_SMMU_FSR_EF)
1299 type = "EXTERNAL";
1300
e25e92e0
RC		 1301 block = a6xx_fault_block(gpu, info->fsynr1 & 0xff);
1302
2a574cc0
JC		 1303 pr_warn_ratelimited("*** gpu fault: ttbr0=%.16llx iova=%.16lx dir=%s type=%s source=%s (%u,%u,%u,%u)\n",
1304 info->ttbr0, iova,
e25e92e0
RC		 1305 flags & IOMMU_FAULT_WRITE ? "WRITE" : "READ",
1306 type, block,
2a574cc0
JC		 1307 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),
1308 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),
1309 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),
1310 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)));
1311
e25e92e0
RC		 1312 if (do_devcoredump) {
1313 /* Turn off the hangcheck timer to keep it from bothering us */
1314 del_timer(&gpu->hangcheck_timer);
1315
1316 gpu->fault_info.ttbr0 = info->ttbr0;
1317 gpu->fault_info.iova = iova;
1318 gpu->fault_info.flags = flags;
1319 gpu->fault_info.type = type;
1320 gpu->fault_info.block = block;
1321
1322 kthread_queue_work(gpu->worker, &gpu->fault_work);
1323 }
1324
2a574cc0 1325 return 0;
4b565ca5
JC		 1326}
1327
1328static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu)
1329{
1330 u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS);
1331
1332 if (status & A6XX_CP_INT_CP_OPCODE_ERROR) {
1333 u32 val;
1334
1335 gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1);
1336 val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA);
1337 dev_err_ratelimited(&gpu->pdev->dev,
1338 "CP | opcode error | possible opcode=0x%8.8X\n",
1339 val);
1340 }
1341
1342 if (status & A6XX_CP_INT_CP_UCODE_ERROR)
1343 dev_err_ratelimited(&gpu->pdev->dev,
1344 "CP ucode error interrupt\n");
1345
1346 if (status & A6XX_CP_INT_CP_HW_FAULT_ERROR)
1347 dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n",
1348 gpu_read(gpu, REG_A6XX_CP_HW_FAULT));
1349
1350 if (status & A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
1351 u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS);
1352
1353 dev_err_ratelimited(&gpu->pdev->dev,
1354 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
1355 val & (1 << 20) ? "READ" : "WRITE",
1356 (val & 0x3ffff), val);
1357 }
1358
1359 if (status & A6XX_CP_INT_CP_AHB_ERROR)
1360 dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n");
1361
1362 if (status & A6XX_CP_INT_CP_VSD_PARITY_ERROR)
1363 dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n");
1364
1365 if (status & A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR)
1366 dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n");
1367
1368}
1369
1370static void a6xx_fault_detect_irq(struct msm_gpu *gpu)
1371{
1372 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1373 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
4b565ca5
JC		 1374 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
1375
e25e92e0
RC		 1376 /*
1377 * If stalled on SMMU fault, we could trip the GPU's hang detection,
1378 * but the fault handler will trigger the devcore dump, and we want
1379 * to otherwise resume normally rather than killing the submit, so
1380 * just bail.
1381 */
1382 if (gpu_read(gpu, REG_A6XX_RBBM_STATUS3) & A6XX_RBBM_STATUS3_SMMU_STALLED_ON_FAULT)
1383 return;
1384
4b565ca5
JC		 1385 /*
1386 * Force the GPU to stay on until after we finish
1387 * collecting information
1388 */
1389 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1);
1390
1391 DRM_DEV_ERROR(&gpu->pdev->dev,
1392 "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
1393 ring ? ring->id : -1, ring ? ring->seqno : 0,
1394 gpu_read(gpu, REG_A6XX_RBBM_STATUS),
1395 gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
1396 gpu_read(gpu, REG_A6XX_CP_RB_WPTR),
1397 gpu_read64(gpu, REG_A6XX_CP_IB1_BASE, REG_A6XX_CP_IB1_BASE_HI),
1398 gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),
1399 gpu_read64(gpu, REG_A6XX_CP_IB2_BASE, REG_A6XX_CP_IB2_BASE_HI),
1400 gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE));
1401
1402 /* Turn off the hangcheck timer to keep it from bothering us */
1403 del_timer(&gpu->hangcheck_timer);
1404
7e688294 1405 kthread_queue_work(gpu->worker, &gpu->recover_work);
4b565ca5
JC		 1406}
1407
1408static irqreturn_t a6xx_irq(struct msm_gpu *gpu)
1409{
5edf2750 1410 struct msm_drm_private *priv = gpu->dev->dev_private;
4b565ca5
JC		 1411 u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS);
1412
1413 gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status);
1414
5edf2750
RC		 1415 if (priv->disable_err_irq)
1416 status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS;
1417
4b565ca5
JC		 1418 if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT)
1419 a6xx_fault_detect_irq(gpu);
1420
1421 if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR)
1422 dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n");
1423
1424 if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR)
1425 a6xx_cp_hw_err_irq(gpu);
1426
1427 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW)
1428 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n");
1429
1430 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
1431 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n");
1432
1433 if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
1434 dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");
1435
1436 if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
1437 msm_gpu_retire(gpu);
1438
1439 return IRQ_HANDLED;
1440}
1441
474dadb8
SM		 1442static void a6xx_llc_rmw(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 mask, u32 or)
1443{
1444 return msm_rmw(a6xx_gpu->llc_mmio + (reg << 2), mask, or);
1445}
1446
1447static void a6xx_llc_write(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 value)
1448{
1449 return msm_writel(value, a6xx_gpu->llc_mmio + (reg << 2));
1450}
1451
1452static void a6xx_llc_deactivate(struct a6xx_gpu *a6xx_gpu)
1453{
1454 llcc_slice_deactivate(a6xx_gpu->llc_slice);
1455 llcc_slice_deactivate(a6xx_gpu->htw_llc_slice);
1456}
1457
1458static void a6xx_llc_activate(struct a6xx_gpu *a6xx_gpu)
1459{
3d247123
JC		 1460 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1461 struct msm_gpu *gpu = &adreno_gpu->base;
9ba873e6 1462 u32 cntl1_regval = 0;
474dadb8
SM		 1463
1464 if (IS_ERR(a6xx_gpu->llc_mmio))
1465 return;
1466
1467 if (!llcc_slice_activate(a6xx_gpu->llc_slice)) {
9ba873e6 1468 u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice);
474dadb8
SM		 1469
1470 gpu_scid &= 0x1f;
1471 cntl1_regval = (gpu_scid << 0) | (gpu_scid << 5) | (gpu_scid << 10) |
1472 (gpu_scid << 15) | (gpu_scid << 20);
9ba873e6
AO		 1473
1474 /* On A660, the SCID programming for UCHE traffic is done in
1475 * A6XX_GBIF_SCACHE_CNTL0[14:10]
1476 */
1477 if (adreno_is_a660_family(adreno_gpu))
1478 gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL0, (0x1f << 10) |
1479 (1 << 8), (gpu_scid << 10) | (1 << 8));
474dadb8
SM		 1480 }
1481
3d247123
JC		 1482 /*
1483 * For targets with a MMU500, activate the slice but don't program the
1484 * register. The XBL will take care of that.
1485 */
474dadb8 1486 if (!llcc_slice_activate(a6xx_gpu->htw_llc_slice)) {
3d247123
JC		 1487 if (!a6xx_gpu->have_mmu500) {
1488 u32 gpuhtw_scid = llcc_get_slice_id(a6xx_gpu->htw_llc_slice);
474dadb8 1489
3d247123
JC		 1490 gpuhtw_scid &= 0x1f;
1491 cntl1_regval |= FIELD_PREP(GENMASK(29, 25), gpuhtw_scid);
1492 }
474dadb8
SM		 1493 }
1494
a6f24383
AO		 1495 if (!cntl1_regval)
1496 return;
1497
1498 /*
1499 * Program the slice IDs for the various GPU blocks and GPU MMU
1500 * pagetables
1501 */
1502 if (!a6xx_gpu->have_mmu500) {
1503 a6xx_llc_write(a6xx_gpu,
1504 REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_1, cntl1_regval);
1505
474dadb8 1506 /*
a6f24383
AO		 1507 * Program cacheability overrides to not allocate cache
1508 * lines on a write miss
474dadb8 1509 */
a6f24383
AO		 1510 a6xx_llc_rmw(a6xx_gpu,
1511 REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_0, 0xF, 0x03);
1512 return;
474dadb8 1513 }
a6f24383
AO		 1514
1515 gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL1, GENMASK(24, 0), cntl1_regval);
474dadb8
SM		 1516}
1517
1518static void a6xx_llc_slices_destroy(struct a6xx_gpu *a6xx_gpu)
1519{
1520 llcc_slice_putd(a6xx_gpu->llc_slice);
1521 llcc_slice_putd(a6xx_gpu->htw_llc_slice);
1522}
1523
1524static void a6xx_llc_slices_init(struct platform_device *pdev,
1525 struct a6xx_gpu *a6xx_gpu)
1526{
3d247123
JC		 1527 struct device_node *phandle;
1528
3d247123
JC		 1529 /*
1530 * There is a different programming path for targets with an mmu500
1531 * attached, so detect if that is the case
1532 */
1533 phandle = of_parse_phandle(pdev->dev.of_node, "iommus", 0);
1534 a6xx_gpu->have_mmu500 = (phandle &&
1535 of_device_is_compatible(phandle, "arm,mmu-500"));
1536 of_node_put(phandle);
1537
4b95d371
JM		 1538 if (a6xx_gpu->have_mmu500)
1539 a6xx_gpu->llc_mmio = NULL;
1540 else
c0e745d7 1541 a6xx_gpu->llc_mmio = msm_ioremap(pdev, "cx_mem");
4b95d371 1542
474dadb8
SM		 1543 a6xx_gpu->llc_slice = llcc_slice_getd(LLCC_GPU);
1544 a6xx_gpu->htw_llc_slice = llcc_slice_getd(LLCC_GPUHTW);
1545
276619c0 1546 if (IS_ERR_OR_NULL(a6xx_gpu->llc_slice) && IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice))
474dadb8
SM		 1547 a6xx_gpu->llc_mmio = ERR_PTR(-EINVAL);
1548}
1549
4b565ca5
JC		 1550static int a6xx_pm_resume(struct msm_gpu *gpu)
1551{
1552 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1553 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1554 int ret;
1555
4b565ca5
JC		 1556 gpu->needs_hw_init = true;
1557
ec1cb6e4
RC		 1558 trace_msm_gpu_resume(0);
1559
f6f59072 1560 mutex_lock(&a6xx_gpu->gmu.lock);
41570b74 1561 ret = a6xx_gmu_resume(a6xx_gpu);
f6f59072 1562 mutex_unlock(&a6xx_gpu->gmu.lock);
41570b74
JC		 1563 if (ret)
1564 return ret;
1565
af5b4fff 1566 msm_devfreq_resume(gpu);
a2c3c0a5 1567
474dadb8
SM		 1568 a6xx_llc_activate(a6xx_gpu);
1569
41570b74 1570 return 0;
4b565ca5
JC		 1571}
1572
1573static int a6xx_pm_suspend(struct msm_gpu *gpu)
1574{
1575 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1576 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
e8b0b994 1577 int i, ret;
4b565ca5 1578
ec1cb6e4
RC		 1579 trace_msm_gpu_suspend(0);
1580
474dadb8
SM		 1581 a6xx_llc_deactivate(a6xx_gpu);
1582
af5b4fff 1583 msm_devfreq_suspend(gpu);
a2c3c0a5 1584
f6f59072 1585 mutex_lock(&a6xx_gpu->gmu.lock);
e8b0b994 1586 ret = a6xx_gmu_stop(a6xx_gpu);
f6f59072 1587 mutex_unlock(&a6xx_gpu->gmu.lock);
e8b0b994
RC		 1588 if (ret)
1589 return ret;
1590
ce86c239 1591 if (a6xx_gpu->shadow_bo)
e8b0b994
RC		 1592 for (i = 0; i < gpu->nr_rings; i++)
1593 a6xx_gpu->shadow[i] = 0;
1594
860a7b2a
RC		 1595 gpu->suspend_count++;
1596
e8b0b994 1597 return 0;
4b565ca5
JC		 1598}
1599
1600static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
1601{
1602 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1603 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
5f98b33b 1604
f6f59072 1605 mutex_lock(&a6xx_gpu->gmu.lock);
4b565ca5
JC		 1606
1607 /* Force the GPU power on so we can read this register */
7a7cbf2a 1608 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
4b565ca5 1609
9fbd3088 1610 *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
f6f59072 1611 REG_A6XX_CP_ALWAYS_ON_COUNTER_HI);
4b565ca5 1612
7a7cbf2a 1613 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
f6f59072
RC		 1614
1615 mutex_unlock(&a6xx_gpu->gmu.lock);
1616
4b565ca5
JC		 1617 return 0;
1618}
1619
4b565ca5
JC		 1620static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu)
1621{
1622 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1623 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1624
1625 return a6xx_gpu->cur_ring;
1626}
1627
1628static void a6xx_destroy(struct msm_gpu *gpu)
1629{
1630 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1631 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1632
1633 if (a6xx_gpu->sqe_bo) {
7ad0e8cf 1634 msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);
f7d33950 1635 drm_gem_object_put(a6xx_gpu->sqe_bo);
4b565ca5
JC		 1636 }
1637
d3a569fc
JC		 1638 if (a6xx_gpu->shadow_bo) {
1639 msm_gem_unpin_iova(a6xx_gpu->shadow_bo, gpu->aspace);
1640 drm_gem_object_put(a6xx_gpu->shadow_bo);
1641 }
1642
474dadb8
SM		 1643 a6xx_llc_slices_destroy(a6xx_gpu);
1644
4b565ca5
JC		 1645 a6xx_gmu_remove(a6xx_gpu);
1646
1647 adreno_gpu_cleanup(adreno_gpu);
fe7952c6 1648
4b565ca5
JC		 1649 kfree(a6xx_gpu);
1650}
1651
a2c3c0a5
SM		 1652static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu)
1653{
1654 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1655 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
16f37102 1656 u64 busy_cycles, busy_time;
a2c3c0a5 1657
eadf7928
JC		 1658
1659 /* Only read the gpu busy if the hardware is already active */
1660 if (pm_runtime_get_if_in_use(a6xx_gpu->gmu.dev) == 0)
1661 return 0;
1662
a2c3c0a5
SM		 1663 busy_cycles = gmu_read64(&a6xx_gpu->gmu,
1664 REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L,
1665 REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H);
1666
16f37102
SP		 1667 busy_time = (busy_cycles - gpu->devfreq.busy_cycles) * 10;
1668 do_div(busy_time, 192);
a2c3c0a5
SM		 1669
1670 gpu->devfreq.busy_cycles = busy_cycles;
1671
eadf7928
JC		 1672 pm_runtime_put(a6xx_gpu->gmu.dev);
1673
16f37102
SP		 1674 if (WARN_ON(busy_time > ~0LU))
1675 return ~0LU;
1676
1677 return (unsigned long)busy_time;
a2c3c0a5
SM		 1678}
1679
4823c030 1680static void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
f6f59072
RC		 1681{
1682 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1683 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1684
1685 mutex_lock(&a6xx_gpu->gmu.lock);
1686 a6xx_gmu_set_freq(gpu, opp);
1687 mutex_unlock(&a6xx_gpu->gmu.lock);
1688}
1689
45596f25
SPR		 1690static struct msm_gem_address_space *
1691a6xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev)
1692{
1693 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1694 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1695 struct iommu_domain *iommu;
1696 struct msm_mmu *mmu;
1697 struct msm_gem_address_space *aspace;
1698 u64 start, size;
1699
1700 iommu = iommu_domain_alloc(&platform_bus_type);
1701 if (!iommu)
1702 return NULL;
1703
1704 /*
1705 * This allows GPU to set the bus attributes required to use system
1706 * cache on behalf of the iommu page table walker.
1707 */
1708 if (!IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice))
1709 adreno_set_llc_attributes(iommu);
1710
1711 mmu = msm_iommu_new(&pdev->dev, iommu);
1712 if (IS_ERR(mmu)) {
1713 iommu_domain_free(iommu);
1714 return ERR_CAST(mmu);
1715 }
1716
1717 /*
1718 * Use the aperture start or SZ_16M, whichever is greater. This will
1719 * ensure that we align with the allocated pagetable range while still
1720 * allowing room in the lower 32 bits for GMEM and whatnot
1721 */
1722 start = max_t(u64, SZ_16M, iommu->geometry.aperture_start);
1723 size = iommu->geometry.aperture_end - start + 1;
1724
1725 aspace = msm_gem_address_space_create(mmu, "gpu",
1726 start & GENMASK_ULL(48, 0), size);
1727
1728 if (IS_ERR(aspace) && !IS_ERR(mmu))
1729 mmu->funcs->destroy(mmu);
1730
1731 return aspace;
1732}
1733
84c31ee1
JC		 1734static struct msm_gem_address_space *
1735a6xx_create_private_address_space(struct msm_gpu *gpu)
1736{
1737 struct msm_mmu *mmu;
1738
1739 mmu = msm_iommu_pagetable_create(gpu->aspace->mmu);
1740
1741 if (IS_ERR(mmu))
1742 return ERR_CAST(mmu);
1743
1744 return msm_gem_address_space_create(mmu,
537fef80 1745 "gpu", 0x100000000ULL, SZ_4G);
84c31ee1
JC		 1746}
1747
d3a569fc
JC		 1748static uint32_t a6xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
1749{
1750 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1751 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1752
1753 if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami)
1754 return a6xx_gpu->shadow[ring->id];
1755
1756 return ring->memptrs->rptr = gpu_read(gpu, REG_A6XX_CP_RB_RPTR);
1757}
1758
fe7952c6
AO		 1759static u32 a618_get_speed_bin(u32 fuse)
1760{
1761 if (fuse == 0)
1762 return 0;
1763 else if (fuse == 169)
1764 return 1;
1765 else if (fuse == 174)
1766 return 2;
1767
1768 return UINT_MAX;
1769}
1770
c43de1aa
AO		 1771static u32 adreno_7c3_get_speed_bin(u32 fuse)
1772{
1773 if (fuse == 0)
1774 return 0;
1775 else if (fuse == 117)
1776 return 0;
1777 else if (fuse == 190)
1778 return 1;
1779
1780 return UINT_MAX;
1781}
1782
27514ce2 1783static u32 fuse_to_supp_hw(struct device *dev, struct adreno_rev rev, u32 fuse)
fe7952c6
AO		 1784{
1785 u32 val = UINT_MAX;
1786
27514ce2 1787 if (adreno_cmp_rev(ADRENO_REV(6, 1, 8, ANY_ID), rev))
fe7952c6
AO		 1788 val = a618_get_speed_bin(fuse);
1789
c43de1aa
AO		 1790 if (adreno_cmp_rev(ADRENO_REV(6, 3, 5, ANY_ID), rev))
1791 val = adreno_7c3_get_speed_bin(fuse);
1792
fe7952c6
AO		 1793 if (val == UINT_MAX) {
1794 DRM_DEV_ERROR(dev,
1795 "missing support for speed-bin: %u. Some OPPs may not be supported by hardware",
1796 fuse);
1797 return UINT_MAX;
1798 }
1799
1800 return (1 << val);
1801}
1802
27514ce2 1803static int a6xx_set_supported_hw(struct device *dev, struct adreno_rev rev)
fe7952c6 1804{
fe7952c6 1805 u32 supp_hw = UINT_MAX;
c9f737c7 1806 u32 speedbin;
7bf168c8 1807 int ret;
fe7952c6 1808
afab9d91 1809 ret = adreno_read_speedbin(dev, &speedbin);
2b0b219e
JS		 1810 /*
1811 * -ENOENT means that the platform doesn't support speedbin which is
1812 * fine
1813 */
1814 if (ret == -ENOENT) {
1815 return 0;
1816 } else if (ret) {
fe7952c6 1817 DRM_DEV_ERROR(dev,
7bf168c8
DA		 1818 "failed to read speed-bin (%d). Some OPPs may not be supported by hardware",
1819 ret);
fe7952c6
AO		 1820 goto done;
1821 }
1822
27514ce2 1823 supp_hw = fuse_to_supp_hw(dev, rev, speedbin);
fe7952c6
AO		 1824
1825done:
11120e93
YL		 1826 ret = devm_pm_opp_set_supported_hw(dev, &supp_hw, 1);
1827 if (ret)
1828 return ret;
fe7952c6 1829
fe7952c6
AO		 1830 return 0;
1831}
1832
4b565ca5
JC		 1833static const struct adreno_gpu_funcs funcs = {
1834 .base = {
1835 .get_param = adreno_get_param,
f7ddbf55 1836 .set_param = adreno_set_param,
4b565ca5
JC		 1837 .hw_init = a6xx_hw_init,
1838 .pm_suspend = a6xx_pm_suspend,
1839 .pm_resume = a6xx_pm_resume,
1840 .recover = a6xx_recover,
1841 .submit = a6xx_submit,
4b565ca5
JC		 1842 .active_ring = a6xx_active_ring,
1843 .irq = a6xx_irq,
1844 .destroy = a6xx_destroy,
b02872df 1845#if defined(CONFIG_DRM_MSM_GPU_STATE)
4b565ca5
JC		 1846 .show = a6xx_show,
1847#endif
a2c3c0a5
SM		 1848 .gpu_busy = a6xx_gpu_busy,
1849 .gpu_get_freq = a6xx_gmu_get_freq,
f6f59072 1850 .gpu_set_freq = a6xx_gpu_set_freq,
b02872df 1851#if defined(CONFIG_DRM_MSM_GPU_STATE)
1707add8
JC		 1852 .gpu_state_get = a6xx_gpu_state_get,
1853 .gpu_state_put = a6xx_gpu_state_put,
b02872df 1854#endif
45596f25 1855 .create_address_space = a6xx_create_address_space,
84c31ee1 1856 .create_private_address_space = a6xx_create_private_address_space,
d3a569fc 1857 .get_rptr = a6xx_get_rptr,
4b565ca5
JC		 1858 },
1859 .get_timestamp = a6xx_get_timestamp,
1860};
1861
1862struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
1863{
1864 struct msm_drm_private *priv = dev->dev_private;
1865 struct platform_device *pdev = priv->gpu_pdev;
e9ba8d55
JC		 1866 struct adreno_platform_config *config = pdev->dev.platform_data;
1867 const struct adreno_info *info;
4b565ca5
JC		 1868 struct device_node *node;
1869 struct a6xx_gpu *a6xx_gpu;
1870 struct adreno_gpu *adreno_gpu;
1871 struct msm_gpu *gpu;
1872 int ret;
1873
1874 a6xx_gpu = kzalloc(sizeof(*a6xx_gpu), GFP_KERNEL);
1875 if (!a6xx_gpu)
1876 return ERR_PTR(-ENOMEM);
1877
1878 adreno_gpu = &a6xx_gpu->base;
1879 gpu = &adreno_gpu->base;
1880
1707add8 1881 adreno_gpu->registers = NULL;
4b565ca5 1882
e9ba8d55
JC		 1883 /*
1884 * We need to know the platform type before calling into adreno_gpu_init
1885 * so that the hw_apriv flag can be correctly set. Snoop into the info
1886 * and grab the revision number
1887 */
1888 info = adreno_info(config->rev);
1889
192f4ee3
AO		 1890 if (info && (info->revn == 650 || info->revn == 660 ||
1891 adreno_cmp_rev(ADRENO_REV(6, 3, 5, ANY_ID), info->rev)))
604234f3
JC		 1892 adreno_gpu->base.hw_apriv = true;
1893
5ca6779d
RC		 1894 /*
1895 * For now only clamp to idle freq for devices where this is known not
1896 * to cause power supply issues:
1897 */
1898 if (info && (info->revn == 618))
1899 gpu->clamp_to_idle = true;
1900
474dadb8
SM		 1901 a6xx_llc_slices_init(pdev, a6xx_gpu);
1902
27514ce2 1903 ret = a6xx_set_supported_hw(&pdev->dev, config->rev);
fe7952c6
AO		 1904 if (ret) {
1905 a6xx_destroy(&(a6xx_gpu->base.base));
1906 return ERR_PTR(ret);
1907 }
1908
4b565ca5
JC		 1909 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1);
1910 if (ret) {
1911 a6xx_destroy(&(a6xx_gpu->base.base));
1912 return ERR_PTR(ret);
1913 }
1914
1915 /* Check if there is a GMU phandle and set it up */
06feed56 1916 node = of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0);
4b565ca5
JC		 1917
1918 /* FIXME: How do we gracefully handle this? */
1919 BUG_ON(!node);
1920
981f2aab 1921 ret = a6xx_gmu_init(a6xx_gpu, node);
4b565ca5
JC		 1922 if (ret) {
1923 a6xx_destroy(&(a6xx_gpu->base.base));
1924 return ERR_PTR(ret);
1925 }
1926
1927 if (gpu->aspace)
1928 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu,
1929 a6xx_fault_handler);
1930
1931 return gpu;
1932}
Linux 6.x block layer and io_uring tree(s)