projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/amdgpu: getting fan speed pwm for vega10 properly
[linux-block.git] / drivers / gpu / drm / amd / amdgpu / vcn_v4_0.c
CommitLineData
8da1170a
LL		 1/*
2 * Copyright 2021 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/firmware.h>
25#include "amdgpu.h"
26#include "amdgpu_vcn.h"
27#include "amdgpu_pm.h"
0b15205c 28#include "amdgpu_cs.h"
8da1170a
LL		 29#include "soc15.h"
30#include "soc15d.h"
31#include "soc15_hw_ip.h"
32#include "vcn_v2_0.h"
aa44beb5 33#include "mmsch_v4_0.h"
8da1170a
LL		 34
35#include "vcn/vcn_4_0_0_offset.h"
36#include "vcn/vcn_4_0_0_sh_mask.h"
37#include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
38
39#include <drm/drm_drv.h>
40
41#define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL
42#define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX
43#define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA
44#define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX
45
46#define VCN_VID_SOC_ADDRESS_2_0 0x1fb00
47#define VCN1_VID_SOC_ADDRESS_3_0 0x48300
48
aa44beb5
JJ		 49#define VCN_HARVEST_MMSCH 0
50
0b15205c
SJ		 51#define RDECODE_MSG_CREATE 0x00000000
52#define RDECODE_MESSAGE_CREATE 0x00000001
53
8da1170a
LL		 54static int amdgpu_ih_clientid_vcns[] = {
55 SOC15_IH_CLIENTID_VCN,
56 SOC15_IH_CLIENTID_VCN1
57};
58
aa44beb5 59static int vcn_v4_0_start_sriov(struct amdgpu_device *adev);
bb4f196b 60static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev);
8da1170a
LL		 61static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev);
62static int vcn_v4_0_set_powergating_state(void *handle,
63 enum amd_powergating_state state);
64static int vcn_v4_0_pause_dpg_mode(struct amdgpu_device *adev,
65 int inst_idx, struct dpg_pause_state *new_state);
aa44beb5 66static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring);
8da1170a
LL		 67
68/**
69 * vcn_v4_0_early_init - set function pointers
70 *
71 * @handle: amdgpu_device pointer
72 *
73 * Set ring and irq function pointers
74 */
75static int vcn_v4_0_early_init(void *handle)
76{
77 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
78
aa44beb5
JJ		 79 if (amdgpu_sriov_vf(adev))
80 adev->vcn.harvest_config = VCN_HARVEST_MMSCH;
81
bb4f196b
RD		 82 /* re-use enc ring as unified ring */
83 adev->vcn.num_enc_rings = 1;
8da1170a 84
bb4f196b 85 vcn_v4_0_set_unified_ring_funcs(adev);
8da1170a
LL		 86 vcn_v4_0_set_irq_funcs(adev);
87
88 return 0;
89}
90
8da1170a
LL		 91/**
92 * vcn_v4_0_sw_init - sw init for VCN block
93 *
94 * @handle: amdgpu_device pointer
95 *
96 * Load firmware and sw initialization
97 */
98static int vcn_v4_0_sw_init(void *handle)
99{
100 struct amdgpu_ring *ring;
8da1170a 101 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
bb4f196b 102 int i, r;
aa44beb5 103 int vcn_doorbell_index = 0;
8da1170a
LL		 104
105 r = amdgpu_vcn_sw_init(adev);
106 if (r)
107 return r;
108
bb4f196b 109 amdgpu_vcn_setup_ucode(adev);
8da1170a
LL		 110
111 r = amdgpu_vcn_resume(adev);
112 if (r)
113 return r;
114
aa44beb5
JJ		 115 if (amdgpu_sriov_vf(adev)) {
116 vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1 - MMSCH_DOORBELL_OFFSET;
117 /* get DWORD offset */
118 vcn_doorbell_index = vcn_doorbell_index << 1;
119 }
120
8da1170a
LL		 121 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
122 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
bb4f196b 123
8da1170a
LL		 124 if (adev->vcn.harvest_config & (1 << i))
125 continue;
bb4f196b
RD		 126
127 atomic_set(&adev->vcn.inst[i].sched_score, 0);
128
129 /* VCN UNIFIED TRAP */
8da1170a 130 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
bb4f196b 131 VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
8da1170a
LL		 132 if (r)
133 return r;
134
bb4f196b
RD		 135 ring = &adev->vcn.inst[i].ring_enc[0];
136 ring->use_doorbell = true;
aa44beb5
JJ		 137 if (amdgpu_sriov_vf(adev))
138 ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1) + 1;
139 else
140 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + 8 * i;
8da1170a 141
bb4f196b 142 sprintf(ring->name, "vcn_unified_%d", i);
8da1170a 143
bb4f196b 144 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
e751e4be 145 AMDGPU_RING_PRIO_0, &adev->vcn.inst[i].sched_score);
bb4f196b
RD		 146 if (r)
147 return r;
8da1170a 148
bb4f196b
RD		 149 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
150 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
151 fw_shared->sq.is_enabled = 1;
8da1170a 152
aa44beb5
JJ		 153 if (amdgpu_sriov_vf(adev))
154 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
155
8da1170a
LL		 156 if (amdgpu_vcnfw_log)
157 amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
158 }
159
aa44beb5
JJ		 160 if (amdgpu_sriov_vf(adev)) {
161 r = amdgpu_virt_alloc_mm_table(adev);
162 if (r)
163 return r;
164 }
165
bb4f196b
RD		 166 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
167 adev->vcn.pause_dpg_mode = vcn_v4_0_pause_dpg_mode;
168
8da1170a
LL		 169 return 0;
170}
171
172/**
173 * vcn_v4_0_sw_fini - sw fini for VCN block
174 *
175 * @handle: amdgpu_device pointer
176 *
177 * VCN suspend and free up sw allocation
178 */
179static int vcn_v4_0_sw_fini(void *handle)
180{
181 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
182 int i, r, idx;
183
8585732b 184 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
bb4f196b
RD		 185 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
186 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
8da1170a 187
bb4f196b
RD		 188 if (adev->vcn.harvest_config & (1 << i))
189 continue;
8da1170a 190
bb4f196b
RD		 191 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
192 fw_shared->present_flag_0 = 0;
193 fw_shared->sq.is_enabled = 0;
194 }
8da1170a 195
bb4f196b
RD		 196 drm_dev_exit(idx);
197 }
8da1170a 198
aa44beb5
JJ		 199 if (amdgpu_sriov_vf(adev))
200 amdgpu_virt_free_mm_table(adev);
201
8da1170a
LL		 202 r = amdgpu_vcn_suspend(adev);
203 if (r)
204 return r;
205
206 r = amdgpu_vcn_sw_fini(adev);
207
208 return r;
209}
210
211/**
212 * vcn_v4_0_hw_init - start and test VCN block
213 *
214 * @handle: amdgpu_device pointer
215 *
216 * Initialize the hardware, boot up the VCPU and do some testing
217 */
218static int vcn_v4_0_hw_init(void *handle)
219{
220 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
221 struct amdgpu_ring *ring;
bb4f196b 222 int i, r;
8da1170a 223
aa44beb5
JJ		 224 if (amdgpu_sriov_vf(adev)) {
225 r = vcn_v4_0_start_sriov(adev);
226 if (r)
227 goto done;
bb4f196b 228
aa44beb5
JJ		 229 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
230 if (adev->vcn.harvest_config & (1 << i))
231 continue;
8da1170a 232
aa44beb5
JJ		 233 ring = &adev->vcn.inst[i].ring_enc[0];
234 if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {
235 ring->sched.ready = false;
236 ring->no_scheduler = true;
237 dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
238 } else {
239 ring->wptr = 0;
240 ring->wptr_old = 0;
241 vcn_v4_0_unified_ring_set_wptr(ring);
242 ring->sched.ready = true;
243 }
244 }
245 } else {
246 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
247 if (adev->vcn.harvest_config & (1 << i))
248 continue;
8da1170a 249
aa44beb5
JJ		 250 ring = &adev->vcn.inst[i].ring_enc[0];
251
252 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
253 ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
254
255 r = amdgpu_ring_test_helper(ring);
256 if (r)
257 goto done;
258
259 }
8da1170a
LL		 260 }
261
262done:
263 if (!r)
264 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
265 (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
266
267 return r;
268}
269
270/**
271 * vcn_v4_0_hw_fini - stop the hardware block
272 *
273 * @handle: amdgpu_device pointer
274 *
275 * Stop the VCN block, mark ring as not ready any more
276 */
277static int vcn_v4_0_hw_fini(void *handle)
278{
279 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
280 int i;
281
282 cancel_delayed_work_sync(&adev->vcn.idle_work);
283
284 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
285 if (adev->vcn.harvest_config & (1 << i))
286 continue;
aa44beb5
JJ		 287 if (!amdgpu_sriov_vf(adev)) {
288 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
8da1170a
LL		 289 (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
290 RREG32_SOC15(VCN, i, regUVD_STATUS))) {
291 vcn_v4_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
aa44beb5 292 }
8da1170a 293 }
aa44beb5 294
8da1170a
LL		 295 }
296
297 return 0;
298}
299
300/**
301 * vcn_v4_0_suspend - suspend VCN block
302 *
303 * @handle: amdgpu_device pointer
304 *
305 * HW fini and suspend VCN block
306 */
307static int vcn_v4_0_suspend(void *handle)
308{
309 int r;
310 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
311
312 r = vcn_v4_0_hw_fini(adev);
313 if (r)
314 return r;
315
316 r = amdgpu_vcn_suspend(adev);
317
318 return r;
319}
320
321/**
322 * vcn_v4_0_resume - resume VCN block
323 *
324 * @handle: amdgpu_device pointer
325 *
326 * Resume firmware and hw init VCN block
327 */
328static int vcn_v4_0_resume(void *handle)
329{
330 int r;
331 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
332
333 r = amdgpu_vcn_resume(adev);
334 if (r)
335 return r;
336
337 r = vcn_v4_0_hw_init(adev);
338
339 return r;
340}
341
342/**
343 * vcn_v4_0_mc_resume - memory controller programming
344 *
345 * @adev: amdgpu_device pointer
346 * @inst: instance number
347 *
348 * Let the VCN memory controller know it's offsets
349 */
350static void vcn_v4_0_mc_resume(struct amdgpu_device *adev, int inst)
351{
352 uint32_t offset, size;
353 const struct common_firmware_header *hdr;
354
355 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
356 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
357
358 /* cache window 0: fw */
359 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
360 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
361 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
362 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
363 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
364 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, 0);
365 offset = 0;
366 } else {
367 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
368 lower_32_bits(adev->vcn.inst[inst].gpu_addr));
369 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
370 upper_32_bits(adev->vcn.inst[inst].gpu_addr));
371 offset = size;
372 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
373 }
374 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE0, size);
375
376 /* cache window 1: stack */
377 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
378 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
379 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
380 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
381 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET1, 0);
382 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
383
384 /* cache window 2: context */
385 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
386 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
387 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
388 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
389 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET2, 0);
390 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
391
392 /* non-cache window */
393 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
394 lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
395 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
396 upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
397 WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
398 WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,
399 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
400}
401
402/**
403 * vcn_v4_0_mc_resume_dpg_mode - memory controller programming for dpg mode
404 *
405 * @adev: amdgpu_device pointer
406 * @inst_idx: instance number index
407 * @indirect: indirectly write sram
408 *
409 * Let the VCN memory controller know it's offsets with dpg mode
410 */
411static void vcn_v4_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
412{
413 uint32_t offset, size;
414 const struct common_firmware_header *hdr;
415 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
416 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
417
418 /* cache window 0: fw */
419 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
420 if (!indirect) {
421 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
422 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
423 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
424 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
425 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
426 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
427 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
428 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
429 } else {
430 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
431 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
432 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
433 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
434 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
435 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
436 }
437 offset = 0;
438 } else {
439 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
440 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
441 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
442 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
443 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
444 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
445 offset = size;
446 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
447 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0),
448 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
8da1170a
LL		 449 }
450
451 if (!indirect)
452 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
453 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
454 else
455 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
456 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
457
458 /* cache window 1: stack */
459 if (!indirect) {
460 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
461 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
462 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
463 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
464 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
465 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
466 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
467 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
468 } else {
469 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
470 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
471 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
472 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
473 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
474 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
475 }
476 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
477 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
478
479 /* cache window 2: context */
480 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
481 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
482 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
483 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
484 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
485 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
486 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
487 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
488 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
489 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
490
491 /* non-cache window */
492 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
493 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
494 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
495 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
496 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
497 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
498 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
499 VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
500 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
501 VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),
502 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect);
503
504 /* VCN global tiling registers */
505 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
506 VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
507}
508
509/**
510 * vcn_v4_0_disable_static_power_gating - disable VCN static power gating
511 *
512 * @adev: amdgpu_device pointer
513 * @inst: instance number
514 *
515 * Disable static power gating for VCN block
516 */
517static void vcn_v4_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
518{
519 uint32_t data = 0;
520
521 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
522 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
523 | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
524 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
525 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
526 | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
527 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
528 | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
529 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
530 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
531 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
532 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
533 | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
534 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
535 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
536
537 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
538 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS,
539 UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
540 } else {
541 uint32_t value;
542
543 value = (inst) ? 0x2200800 : 0;
544 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
545 | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
546 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
547 | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
548 | 1 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
549 | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
550 | 1 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
551 | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
552 | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
553 | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
554 | 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
555 | 1 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
556 | 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
557 | 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
558
559 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
560 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, value, 0x3F3FFFFF);
561 }
562
563 data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
564 data &= ~0x103;
565 if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
566 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
567 UVD_POWER_STATUS__UVD_PG_EN_MASK;
568
569 WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
570
571 return;
572}
573
574/**
575 * vcn_v4_0_enable_static_power_gating - enable VCN static power gating
576 *
577 * @adev: amdgpu_device pointer
578 * @inst: instance number
579 *
580 * Enable static power gating for VCN block
581 */
582static void vcn_v4_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
583{
584 uint32_t data;
585
586 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
587 /* Before power off, this indicator has to be turned on */
588 data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
589 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
590 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
591 WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
592
593 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
594 | 2 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
595 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
596 | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
597 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
598 | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
599 | 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
600 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
601 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
602 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
603 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
604 | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
605 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
606 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
607 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
608
609 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
610 | 2 << UVD_PGFSM_STATUS__UVDS_PWR_STATUS__SHIFT
611 | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
612 | 2 << UVD_PGFSM_STATUS__UVDTC_PWR_STATUS__SHIFT
613 | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
614 | 2 << UVD_PGFSM_STATUS__UVDTA_PWR_STATUS__SHIFT
615 | 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
616 | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
617 | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
618 | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
619 | 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
620 | 2 << UVD_PGFSM_STATUS__UVDTB_PWR_STATUS__SHIFT
621 | 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
622 | 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
623 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
624 }
625
626 return;
627}
628
629/**
630 * vcn_v4_0_disable_clock_gating - disable VCN clock gating
631 *
632 * @adev: amdgpu_device pointer
633 * @inst: instance number
634 *
635 * Disable clock gating for VCN block
636 */
637static void vcn_v4_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
638{
639 uint32_t data;
640
641 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
642 return;
643
644 /* VCN disable CGC */
645 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
646 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
647 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
648 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
649 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
650
651 data = RREG32_SOC15(VCN, inst, regUVD_CGC_GATE);
652 data &= ~(UVD_CGC_GATE__SYS_MASK
653 | UVD_CGC_GATE__UDEC_MASK
654 | UVD_CGC_GATE__MPEG2_MASK
655 | UVD_CGC_GATE__REGS_MASK
656 | UVD_CGC_GATE__RBC_MASK
657 | UVD_CGC_GATE__LMI_MC_MASK
658 | UVD_CGC_GATE__LMI_UMC_MASK
659 | UVD_CGC_GATE__IDCT_MASK
660 | UVD_CGC_GATE__MPRD_MASK
661 | UVD_CGC_GATE__MPC_MASK
662 | UVD_CGC_GATE__LBSI_MASK
663 | UVD_CGC_GATE__LRBBM_MASK
664 | UVD_CGC_GATE__UDEC_RE_MASK
665 | UVD_CGC_GATE__UDEC_CM_MASK
666 | UVD_CGC_GATE__UDEC_IT_MASK
667 | UVD_CGC_GATE__UDEC_DB_MASK
668 | UVD_CGC_GATE__UDEC_MP_MASK
669 | UVD_CGC_GATE__WCB_MASK
670 | UVD_CGC_GATE__VCPU_MASK
671 | UVD_CGC_GATE__MMSCH_MASK);
672
673 WREG32_SOC15(VCN, inst, regUVD_CGC_GATE, data);
674 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF);
675
676 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
677 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
678 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
679 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
680 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
681 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
682 | UVD_CGC_CTRL__SYS_MODE_MASK
683 | UVD_CGC_CTRL__UDEC_MODE_MASK
684 | UVD_CGC_CTRL__MPEG2_MODE_MASK
685 | UVD_CGC_CTRL__REGS_MODE_MASK
686 | UVD_CGC_CTRL__RBC_MODE_MASK
687 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
688 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
689 | UVD_CGC_CTRL__IDCT_MODE_MASK
690 | UVD_CGC_CTRL__MPRD_MODE_MASK
691 | UVD_CGC_CTRL__MPC_MODE_MASK
692 | UVD_CGC_CTRL__LBSI_MODE_MASK
693 | UVD_CGC_CTRL__LRBBM_MODE_MASK
694 | UVD_CGC_CTRL__WCB_MODE_MASK
695 | UVD_CGC_CTRL__VCPU_MODE_MASK
696 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
697 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
698
699 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE);
700 data |= (UVD_SUVD_CGC_GATE__SRE_MASK
701 | UVD_SUVD_CGC_GATE__SIT_MASK
702 | UVD_SUVD_CGC_GATE__SMP_MASK
703 | UVD_SUVD_CGC_GATE__SCM_MASK
704 | UVD_SUVD_CGC_GATE__SDB_MASK
705 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
8da1170a
LL		 706 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
707 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
708 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
709 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
710 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
711 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
712 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
713 | UVD_SUVD_CGC_GATE__SCLR_MASK
714 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
715 | UVD_SUVD_CGC_GATE__ENT_MASK
716 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
717 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
718 | UVD_SUVD_CGC_GATE__SITE_MASK
719 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
720 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
721 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
722 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
723 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
724 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE, data);
725
726 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
727 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
728 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
729 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
730 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
731 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
732 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
733 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
734 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
735 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
736 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
737 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
738}
739
740/**
741 * vcn_v4_0_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode
742 *
743 * @adev: amdgpu_device pointer
744 * @sram_sel: sram select
745 * @inst_idx: instance number index
746 * @indirect: indirectly write sram
747 *
748 * Disable clock gating for VCN block with dpg mode
749 */
750static void vcn_v4_0_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel,
751 int inst_idx, uint8_t indirect)
752{
753 uint32_t reg_data = 0;
754
755 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
756 return;
757
758 /* enable sw clock gating control */
759 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
760 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
761 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
762 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
763 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
764 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
765 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
766 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
767 UVD_CGC_CTRL__SYS_MODE_MASK |
768 UVD_CGC_CTRL__UDEC_MODE_MASK |
769 UVD_CGC_CTRL__MPEG2_MODE_MASK |
770 UVD_CGC_CTRL__REGS_MODE_MASK |
771 UVD_CGC_CTRL__RBC_MODE_MASK |
772 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
773 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
774 UVD_CGC_CTRL__IDCT_MODE_MASK |
775 UVD_CGC_CTRL__MPRD_MODE_MASK |
776 UVD_CGC_CTRL__MPC_MODE_MASK |
777 UVD_CGC_CTRL__LBSI_MODE_MASK |
778 UVD_CGC_CTRL__LRBBM_MODE_MASK |
779 UVD_CGC_CTRL__WCB_MODE_MASK |
780 UVD_CGC_CTRL__VCPU_MODE_MASK);
781 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
782 VCN, inst_idx, regUVD_CGC_CTRL), reg_data, sram_sel, indirect);
783
784 /* turn off clock gating */
785 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
786 VCN, inst_idx, regUVD_CGC_GATE), 0, sram_sel, indirect);
787
788 /* turn on SUVD clock gating */
789 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
790 VCN, inst_idx, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
791
792 /* turn on sw mode in UVD_SUVD_CGC_CTRL */
793 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
794 VCN, inst_idx, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
795}
796
797/**
798 * vcn_v4_0_enable_clock_gating - enable VCN clock gating
799 *
800 * @adev: amdgpu_device pointer
801 * @inst: instance number
802 *
803 * Enable clock gating for VCN block
804 */
805static void vcn_v4_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
806{
807 uint32_t data;
808
809 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
810 return;
811
812 /* enable VCN CGC */
813 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
814 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
815 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
816 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
817 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
818
819 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
820 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
821 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
822 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
823 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
824 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
825 | UVD_CGC_CTRL__SYS_MODE_MASK
826 | UVD_CGC_CTRL__UDEC_MODE_MASK
827 | UVD_CGC_CTRL__MPEG2_MODE_MASK
828 | UVD_CGC_CTRL__REGS_MODE_MASK
829 | UVD_CGC_CTRL__RBC_MODE_MASK
830 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
831 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
832 | UVD_CGC_CTRL__IDCT_MODE_MASK
833 | UVD_CGC_CTRL__MPRD_MODE_MASK
834 | UVD_CGC_CTRL__MPC_MODE_MASK
835 | UVD_CGC_CTRL__LBSI_MODE_MASK
836 | UVD_CGC_CTRL__LRBBM_MODE_MASK
837 | UVD_CGC_CTRL__WCB_MODE_MASK
838 | UVD_CGC_CTRL__VCPU_MODE_MASK
839 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
840 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
841
842 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
843 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
844 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
845 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
846 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
847 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
848 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
849 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
850 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
851 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
852 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
853 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
854
855 return;
856}
857
858/**
859 * vcn_v4_0_start_dpg_mode - VCN start with dpg mode
860 *
861 * @adev: amdgpu_device pointer
862 * @inst_idx: instance number index
863 * @indirect: indirectly write sram
864 *
865 * Start VCN block with dpg mode
866 */
867static int vcn_v4_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
868{
869 volatile struct amdgpu_vcn4_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
870 struct amdgpu_ring *ring;
871 uint32_t tmp;
8da1170a
LL		 872
873 /* disable register anti-hang mechanism */
874 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 1,
875 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
876 /* enable dynamic power gating mode */
877 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS);
878 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
879 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
880 WREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS, tmp);
881
882 if (indirect)
883 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
884
885 /* enable clock gating */
886 vcn_v4_0_disable_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
887
888 /* enable VCPU clock */
889 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
890 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
891 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
892 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
893
894 /* disable master interupt */
895 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
896 VCN, inst_idx, regUVD_MASTINT_EN), 0, 0, indirect);
897
898 /* setup regUVD_LMI_CTRL */
899 tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
900 UVD_LMI_CTRL__REQ_MODE_MASK |
901 UVD_LMI_CTRL__CRC_RESET_MASK |
902 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
903 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
904 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
905 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
906 0x00100000L);
907 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
908 VCN, inst_idx, regUVD_LMI_CTRL), tmp, 0, indirect);
909
910 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
911 VCN, inst_idx, regUVD_MPC_CNTL),
912 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
913
914 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
915 VCN, inst_idx, regUVD_MPC_SET_MUXA0),
916 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
917 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
918 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
919 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
920
921 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
922 VCN, inst_idx, regUVD_MPC_SET_MUXB0),
923 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
924 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
925 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
926 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
927
928 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
929 VCN, inst_idx, regUVD_MPC_SET_MUX),
930 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
931 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
932 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
933
934 vcn_v4_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
935
936 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
937 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
938 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
939 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
940
941 /* enable LMI MC and UMC channels */
942 tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
943 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
944 VCN, inst_idx, regUVD_LMI_CTRL2), tmp, 0, indirect);
945
946 /* enable master interrupt */
947 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
948 VCN, inst_idx, regUVD_MASTINT_EN),
949 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
950
951
952 if (indirect)
953 psp_update_vcn_sram(adev, inst_idx, adev->vcn.inst[inst_idx].dpg_sram_gpu_addr,
954 (uint32_t)((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr -
955 (uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr));
956
bb4f196b 957 ring = &adev->vcn.inst[inst_idx].ring_enc[0];
8da1170a 958
bb4f196b
RD		 959 WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO, ring->gpu_addr);
960 WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
961 WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE, ring->ring_size / 4);
8da1170a 962
8da1170a 963 tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
bb4f196b 964 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
8da1170a 965 WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
bb4f196b
RD		 966 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
967 WREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR, 0);
968 WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, 0);
8da1170a 969
e751e4be 970 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR);
bb4f196b
RD		 971 WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, tmp);
972 ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
8da1170a
LL		 973
974 tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
bb4f196b 975 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
8da1170a 976 WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
bb4f196b 977 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
8da1170a 978
bb4f196b
RD		 979 WREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL,
980 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
981 VCN_RB1_DB_CTRL__EN_MASK);
8da1170a 982
8da1170a
LL		 983 return 0;
984}
985
986
987/**
988 * vcn_v4_0_start - VCN start
989 *
990 * @adev: amdgpu_device pointer
991 *
992 * Start VCN block
993 */
994static int vcn_v4_0_start(struct amdgpu_device *adev)
995{
996 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
997 struct amdgpu_ring *ring;
998 uint32_t tmp;
999 int i, j, k, r;
1000
1001 if (adev->pm.dpm_enabled)
1002 amdgpu_dpm_enable_uvd(adev, true);
1003
1004 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
bb4f196b
RD		 1005 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1006
8da1170a
LL		 1007 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1008 r = vcn_v4_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
1009 continue;
1010 }
1011
1012 /* disable VCN power gating */
1013 vcn_v4_0_disable_static_power_gating(adev, i);
1014
1015 /* set VCN status busy */
1016 tmp = RREG32_SOC15(VCN, i, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1017 WREG32_SOC15(VCN, i, regUVD_STATUS, tmp);
1018
1019 /*SW clock gating */
1020 vcn_v4_0_disable_clock_gating(adev, i);
1021
1022 /* enable VCPU clock */
1023 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
bb4f196b 1024 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
8da1170a
LL		 1025
1026 /* disable master interrupt */
1027 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN), 0,
bb4f196b 1028 ~UVD_MASTINT_EN__VCPU_EN_MASK);
8da1170a
LL		 1029
1030 /* enable LMI MC and UMC channels */
1031 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_LMI_CTRL2), 0,
bb4f196b 1032 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
8da1170a
LL		 1033
1034 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1035 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1036 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1037 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1038
1039 /* setup regUVD_LMI_CTRL */
1040 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL);
1041 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL, tmp |
bb4f196b
RD		 1042 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1043 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1044 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1045 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
8da1170a
LL		 1046
1047 /* setup regUVD_MPC_CNTL */
1048 tmp = RREG32_SOC15(VCN, i, regUVD_MPC_CNTL);
1049 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1050 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1051 WREG32_SOC15(VCN, i, regUVD_MPC_CNTL, tmp);
1052
1053 /* setup UVD_MPC_SET_MUXA0 */
1054 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXA0,
bb4f196b
RD		 1055 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1056 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1057 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1058 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
8da1170a
LL		 1059
1060 /* setup UVD_MPC_SET_MUXB0 */
1061 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXB0,
bb4f196b
RD		 1062 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1063 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1064 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1065 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
8da1170a
LL		 1066
1067 /* setup UVD_MPC_SET_MUX */
1068 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUX,
bb4f196b
RD		 1069 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1070 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1071 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
8da1170a
LL		 1072
1073 vcn_v4_0_mc_resume(adev, i);
1074
1075 /* VCN global tiling registers */
1076 WREG32_SOC15(VCN, i, regUVD_GFX10_ADDR_CONFIG,
bb4f196b 1077 adev->gfx.config.gb_addr_config);
8da1170a
LL		 1078
1079 /* unblock VCPU register access */
1080 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL), 0,
bb4f196b 1081 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
8da1170a
LL		 1082
1083 /* release VCPU reset to boot */
1084 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
bb4f196b 1085 ~UVD_VCPU_CNTL__BLK_RST_MASK);
8da1170a
LL		 1086
1087 for (j = 0; j < 10; ++j) {
1088 uint32_t status;
1089
1090 for (k = 0; k < 100; ++k) {
1091 status = RREG32_SOC15(VCN, i, regUVD_STATUS);
1092 if (status & 2)
1093 break;
1094 mdelay(10);
1095 if (amdgpu_emu_mode==1)
1096 msleep(1);
1097 }
1098
1099 if (amdgpu_emu_mode==1) {
736f7308 1100 r = -1;
8da1170a
LL		 1101 if (status & 2) {
1102 r = 0;
1103 break;
1104 }
1105 } else {
1106 r = 0;
1107 if (status & 2)
1108 break;
1109
bb4f196b
RD		 1110 dev_err(adev->dev, "VCN[%d] is not responding, trying to reset the VCPU!!!\n", i);
1111 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
e751e4be
RD		 1112 UVD_VCPU_CNTL__BLK_RST_MASK,
1113 ~UVD_VCPU_CNTL__BLK_RST_MASK);
8da1170a
LL		 1114 mdelay(10);
1115 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
bb4f196b 1116 ~UVD_VCPU_CNTL__BLK_RST_MASK);
8da1170a
LL		 1117
1118 mdelay(10);
1119 r = -1;
1120 }
1121 }
1122
1123 if (r) {
bb4f196b 1124 dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);
8da1170a
LL		 1125 return r;
1126 }
1127
1128 /* enable master interrupt */
1129 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN),
bb4f196b
RD		 1130 UVD_MASTINT_EN__VCPU_EN_MASK,
1131 ~UVD_MASTINT_EN__VCPU_EN_MASK);
8da1170a
LL		 1132
1133 /* clear the busy bit of VCN_STATUS */
1134 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_STATUS), 0,
bb4f196b 1135 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
8da1170a 1136
bb4f196b
RD		 1137 ring = &adev->vcn.inst[i].ring_enc[0];
1138 WREG32_SOC15(VCN, i, regVCN_RB1_DB_CTRL,
1139 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1140 VCN_RB1_DB_CTRL__EN_MASK);
8da1170a 1141
bb4f196b
RD		 1142 WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO, ring->gpu_addr);
1143 WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1144 WREG32_SOC15(VCN, i, regUVD_RB_SIZE, ring->ring_size / 4);
8da1170a 1145
bb4f196b
RD		 1146 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1147 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1148 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1149 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1150 WREG32_SOC15(VCN, i, regUVD_RB_RPTR, 0);
1151 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, 0);
8da1170a 1152
8da1170a
LL		 1153 tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR);
1154 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, tmp);
1155 ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR);
bb4f196b
RD		 1156
1157 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1158 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1159 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1160 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
8da1170a
LL		 1161 }
1162
1163 return 0;
1164}
1165
aa44beb5
JJ		 1166static int vcn_v4_0_start_sriov(struct amdgpu_device *adev)
1167{
1168 int i;
1169 struct amdgpu_ring *ring_enc;
1170 uint64_t cache_addr;
1171 uint64_t rb_enc_addr;
1172 uint64_t ctx_addr;
1173 uint32_t param, resp, expected;
1174 uint32_t offset, cache_size;
1175 uint32_t tmp, timeout;
1176
1177 struct amdgpu_mm_table *table = &adev->virt.mm_table;
1178 uint32_t *table_loc;
1179 uint32_t table_size;
1180 uint32_t size, size_dw;
1181 uint32_t init_status;
1182 uint32_t enabled_vcn;
1183
1184 struct mmsch_v4_0_cmd_direct_write
1185 direct_wt = { {0} };
1186 struct mmsch_v4_0_cmd_direct_read_modify_write
1187 direct_rd_mod_wt = { {0} };
1188 struct mmsch_v4_0_cmd_end end = { {0} };
1189 struct mmsch_v4_0_init_header header;
1190
1191 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
1192 volatile struct amdgpu_fw_shared_rb_setup *rb_setup;
1193
1194 direct_wt.cmd_header.command_type =
1195 MMSCH_COMMAND__DIRECT_REG_WRITE;
1196 direct_rd_mod_wt.cmd_header.command_type =
1197 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1198 end.cmd_header.command_type =
1199 MMSCH_COMMAND__END;
1200
1201 header.version = MMSCH_VERSION;
1202 header.total_size = sizeof(struct mmsch_v4_0_init_header) >> 2;
1203 for (i = 0; i < AMDGPU_MAX_VCN_INSTANCES; i++) {
1204 header.inst[i].init_status = 0;
1205 header.inst[i].table_offset = 0;
1206 header.inst[i].table_size = 0;
1207 }
1208
1209 table_loc = (uint32_t *)table->cpu_addr;
1210 table_loc += header.total_size;
1211 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1212 if (adev->vcn.harvest_config & (1 << i))
1213 continue;
1214
1215 table_size = 0;
1216
1217 MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1218 regUVD_STATUS),
1219 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1220
1221 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
1222
1223 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1224 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1225 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1226 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1227 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1228 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1229 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1230 offset = 0;
1231 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1232 regUVD_VCPU_CACHE_OFFSET0),
1233 0);
1234 } else {
1235 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1236 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1237 lower_32_bits(adev->vcn.inst[i].gpu_addr));
1238 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1239 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1240 upper_32_bits(adev->vcn.inst[i].gpu_addr));
1241 offset = cache_size;
1242 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1243 regUVD_VCPU_CACHE_OFFSET0),
1244 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1245 }
1246
1247 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1248 regUVD_VCPU_CACHE_SIZE0),
1249 cache_size);
1250
1251 cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1252 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1253 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1254 lower_32_bits(cache_addr));
1255 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1256 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1257 upper_32_bits(cache_addr));
1258 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1259 regUVD_VCPU_CACHE_OFFSET1),
1260 0);
1261 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1262 regUVD_VCPU_CACHE_SIZE1),
1263 AMDGPU_VCN_STACK_SIZE);
1264
1265 cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1266 AMDGPU_VCN_STACK_SIZE;
1267 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1268 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1269 lower_32_bits(cache_addr));
1270 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1271 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1272 upper_32_bits(cache_addr));
1273 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1274 regUVD_VCPU_CACHE_OFFSET2),
1275 0);
1276 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1277 regUVD_VCPU_CACHE_SIZE2),
1278 AMDGPU_VCN_CONTEXT_SIZE);
1279
1280 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1281 rb_setup = &fw_shared->rb_setup;
1282
1283 ring_enc = &adev->vcn.inst[i].ring_enc[0];
1284 ring_enc->wptr = 0;
1285 rb_enc_addr = ring_enc->gpu_addr;
1286
1287 rb_setup->is_rb_enabled_flags |= RB_ENABLED;
1288 rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr);
1289 rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr);
1290 rb_setup->rb_size = ring_enc->ring_size / 4;
1291 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
1292
1293 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1294 regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
1295 lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
1296 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1297 regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
1298 upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
1299 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1300 regUVD_VCPU_NONCACHE_SIZE0),
1301 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
1302
1303 /* add end packet */
1304 MMSCH_V4_0_INSERT_END();
1305
1306 /* refine header */
1307 header.inst[i].init_status = 0;
1308 header.inst[i].table_offset = header.total_size;
1309 header.inst[i].table_size = table_size;
1310 header.total_size += table_size;
1311 }
1312
1313 /* Update init table header in memory */
1314 size = sizeof(struct mmsch_v4_0_init_header);
1315 table_loc = (uint32_t *)table->cpu_addr;
1316 memcpy((void *)table_loc, &header, size);
1317
1318 /* message MMSCH (in VCN[0]) to initialize this client
1319 * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1320 * of memory descriptor location
1321 */
1322 ctx_addr = table->gpu_addr;
1323 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1324 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1325
1326 /* 2, update vmid of descriptor */
1327 tmp = RREG32_SOC15(VCN, 0, regMMSCH_VF_VMID);
1328 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1329 /* use domain0 for MM scheduler */
1330 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1331 WREG32_SOC15(VCN, 0, regMMSCH_VF_VMID, tmp);
1332
1333 /* 3, notify mmsch about the size of this descriptor */
1334 size = header.total_size;
1335 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_SIZE, size);
1336
1337 /* 4, set resp to zero */
1338 WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP, 0);
1339
1340 /* 5, kick off the initialization and wait until
1341 * MMSCH_VF_MAILBOX_RESP becomes non-zero
1342 */
1343 param = 0x00000001;
1344 WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_HOST, param);
1345 tmp = 0;
1346 timeout = 1000;
1347 resp = 0;
1348 expected = MMSCH_VF_MAILBOX_RESP__OK;
1349 while (resp != expected) {
1350 resp = RREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP);
1351 if (resp != 0)
1352 break;
1353
1354 udelay(10);
1355 tmp = tmp + 10;
1356 if (tmp >= timeout) {
1357 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1358 " waiting for regMMSCH_VF_MAILBOX_RESP "\
1359 "(expected=0x%08x, readback=0x%08x)\n",
1360 tmp, expected, resp);
1361 return -EBUSY;
1362 }
1363 }
1364 enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0;
1365 init_status = ((struct mmsch_v4_0_init_header *)(table_loc))->inst[enabled_vcn].init_status;
1366 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE
1367 && init_status != MMSCH_VF_ENGINE_STATUS__PASS)
1368 DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\
1369 "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status);
1370
1371 return 0;
1372}
1373
8da1170a
LL		 1374/**
1375 * vcn_v4_0_stop_dpg_mode - VCN stop with dpg mode
1376 *
1377 * @adev: amdgpu_device pointer
1378 * @inst_idx: instance number index
1379 *
1380 * Stop VCN block with dpg mode
1381 */
385bf5a8 1382static void vcn_v4_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
8da1170a
LL		 1383{
1384 uint32_t tmp;
1385
1386 /* Wait for power status to be 1 */
1387 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1388 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1389
1390 /* wait for read ptr to be equal to write ptr */
1391 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1392 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1393
8da1170a
LL		 1394 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1395 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1396
1397 /* disable dynamic power gating mode */
1398 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 0,
1399 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
8da1170a
LL		 1400}
1401
1402/**
1403 * vcn_v4_0_stop - VCN stop
1404 *
1405 * @adev: amdgpu_device pointer
1406 *
1407 * Stop VCN block
1408 */
1409static int vcn_v4_0_stop(struct amdgpu_device *adev)
1410{
bb4f196b 1411 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
8da1170a
LL		 1412 uint32_t tmp;
1413 int i, r = 0;
1414
1415 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
bb4f196b
RD		 1416 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1417 fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;
1418
8da1170a 1419 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
385bf5a8 1420 vcn_v4_0_stop_dpg_mode(adev, i);
8da1170a
LL		 1421 continue;
1422 }
1423
1424 /* wait for vcn idle */
1425 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1426 if (r)
1427 return r;
1428
1429 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1430 UVD_LMI_STATUS__READ_CLEAN_MASK |
1431 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1432 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1433 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1434 if (r)
1435 return r;
1436
1437 /* disable LMI UMC channel */
1438 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL2);
1439 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1440 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL2, tmp);
1441 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
1442 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1443 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1444 if (r)
1445 return r;
1446
1447 /* block VCPU register access */
1448 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL),
1449 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1450 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1451
1452 /* reset VCPU */
1453 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1454 UVD_VCPU_CNTL__BLK_RST_MASK,
1455 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1456
1457 /* disable VCPU clock */
1458 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1459 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1460
1461 /* apply soft reset */
1462 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1463 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1464 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1465 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1466 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1467 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1468
1469 /* clear status */
1470 WREG32_SOC15(VCN, i, regUVD_STATUS, 0);
1471
1472 /* apply HW clock gating */
1473 vcn_v4_0_enable_clock_gating(adev, i);
1474
1475 /* enable VCN power gating */
1476 vcn_v4_0_enable_static_power_gating(adev, i);
1477 }
1478
1479 if (adev->pm.dpm_enabled)
1480 amdgpu_dpm_enable_uvd(adev, false);
1481
1482 return 0;
1483}
1484
1485/**
1486 * vcn_v4_0_pause_dpg_mode - VCN pause with dpg mode
1487 *
1488 * @adev: amdgpu_device pointer
1489 * @inst_idx: instance number index
1490 * @new_state: pause state
1491 *
1492 * Pause dpg mode for VCN block
1493 */
1494static int vcn_v4_0_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx,
1495 struct dpg_pause_state *new_state)
1496{
1497 uint32_t reg_data = 0;
1498 int ret_code;
1499
1500 /* pause/unpause if state is changed */
1501 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1502 DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",
1503 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1504 reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &
1505 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1506
1507 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1508 ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
1509 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1510
1511 if (!ret_code) {
1512 /* pause DPG */
1513 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1514 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1515
1516 /* wait for ACK */
1517 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,
1518 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1519 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1520
1521 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS,
1522 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1523 }
1524 } else {
1525 /* unpause dpg, no need to wait */
1526 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1527 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
8da1170a
LL		 1528 }
1529 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1530 }
1531
1532 return 0;
1533}
1534
1535/**
bb4f196b 1536 * vcn_v4_0_unified_ring_get_rptr - get unified read pointer
8da1170a
LL		 1537 *
1538 * @ring: amdgpu_ring pointer
1539 *
bb4f196b 1540 * Returns the current hardware unified read pointer
8da1170a 1541 */
bb4f196b 1542static uint64_t vcn_v4_0_unified_ring_get_rptr(struct amdgpu_ring *ring)
8da1170a
LL		 1543{
1544 struct amdgpu_device *adev = ring->adev;
1545
bb4f196b
RD		 1546 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1547 DRM_ERROR("wrong ring id is identified in %s", __func__);
1548
1549 return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR);
8da1170a
LL		 1550}
1551
1552/**
bb4f196b 1553 * vcn_v4_0_unified_ring_get_wptr - get unified write pointer
8da1170a
LL		 1554 *
1555 * @ring: amdgpu_ring pointer
1556 *
bb4f196b 1557 * Returns the current hardware unified write pointer
8da1170a 1558 */
bb4f196b 1559static uint64_t vcn_v4_0_unified_ring_get_wptr(struct amdgpu_ring *ring)
8da1170a
LL		 1560{
1561 struct amdgpu_device *adev = ring->adev;
1562
bb4f196b
RD		 1563 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1564 DRM_ERROR("wrong ring id is identified in %s", __func__);
1565
8da1170a
LL		 1566 if (ring->use_doorbell)
1567 return *ring->wptr_cpu_addr;
1568 else
bb4f196b 1569 return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR);
8da1170a
LL		 1570}
1571
1572/**
bb4f196b 1573 * vcn_v4_0_unified_ring_set_wptr - set enc write pointer
8da1170a
LL		 1574 *
1575 * @ring: amdgpu_ring pointer
1576 *
bb4f196b 1577 * Commits the enc write pointer to the hardware
8da1170a 1578 */
bb4f196b 1579static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring)
8da1170a
LL		 1580{
1581 struct amdgpu_device *adev = ring->adev;
1582
bb4f196b
RD		 1583 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1584 DRM_ERROR("wrong ring id is identified in %s", __func__);
8da1170a
LL		 1585
1586 if (ring->use_doorbell) {
1587 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1588 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1589 } else {
bb4f196b 1590 WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR, lower_32_bits(ring->wptr));
8da1170a
LL		 1591 }
1592}
1593
0b15205c
SJ		 1594static int vcn_v4_0_limit_sched(struct amdgpu_cs_parser *p)
1595{
1596 struct drm_gpu_scheduler **scheds;
1597
1598 /* The create msg must be in the first IB submitted */
1599 if (atomic_read(&p->entity->fence_seq))
1600 return -EINVAL;
1601
1602 scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_ENC]
1603 [AMDGPU_RING_PRIO_0].sched;
1604 drm_sched_entity_modify_sched(p->entity, scheds, 1);
1605 return 0;
1606}
1607
1608static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr)
1609{
1610 struct ttm_operation_ctx ctx = { false, false };
1611 struct amdgpu_bo_va_mapping *map;
1612 uint32_t *msg, num_buffers;
1613 struct amdgpu_bo *bo;
1614 uint64_t start, end;
1615 unsigned int i;
1616 void *ptr;
1617 int r;
1618
1619 addr &= AMDGPU_GMC_HOLE_MASK;
1620 r = amdgpu_cs_find_mapping(p, addr, &bo, &map);
1621 if (r) {
1622 DRM_ERROR("Can't find BO for addr 0x%08llx\n", addr);
1623 return r;
1624 }
1625
1626 start = map->start * AMDGPU_GPU_PAGE_SIZE;
1627 end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1628 if (addr & 0x7) {
1629 DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1630 return -EINVAL;
1631 }
1632
1633 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1634 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1635 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1636 if (r) {
1637 DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1638 return r;
1639 }
1640
1641 r = amdgpu_bo_kmap(bo, &ptr);
1642 if (r) {
1643 DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1644 return r;
1645 }
1646
1647 msg = ptr + addr - start;
1648
1649 /* Check length */
1650 if (msg[1] > end - addr) {
1651 r = -EINVAL;
1652 goto out;
1653 }
1654
1655 if (msg[3] != RDECODE_MSG_CREATE)
1656 goto out;
1657
1658 num_buffers = msg[2];
1659 for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1660 uint32_t offset, size, *create;
1661
1662 if (msg[0] != RDECODE_MESSAGE_CREATE)
1663 continue;
1664
1665 offset = msg[1];
1666 size = msg[2];
1667
1668 if (offset + size > end) {
1669 r = -EINVAL;
1670 goto out;
1671 }
1672
1673 create = ptr + addr + offset - start;
1674
1675 /* H246, HEVC and VP9 can run on any instance */
1676 if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
1677 continue;
1678
1679 r = vcn_v4_0_limit_sched(p);
1680 if (r)
1681 goto out;
1682 }
1683
1684out:
1685 amdgpu_bo_kunmap(bo);
1686 return r;
1687}
1688
1689#define RADEON_VCN_ENGINE_TYPE_DECODE (0x00000003)
1690
1691static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1692 struct amdgpu_job *job,
1693 struct amdgpu_ib *ib)
1694{
1695 struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched);
1696 struct amdgpu_vcn_decode_buffer *decode_buffer = NULL;
1697 uint32_t val;
1698 int r = 0;
1699
1700 /* The first instance can decode anything */
1701 if (!ring->me)
1702 return r;
1703
1704 /* unified queue ib header has 8 double words. */
1705 if (ib->length_dw < 8)
1706 return r;
1707
1708 val = amdgpu_ib_get_value(ib, 6); //RADEON_VCN_ENGINE_TYPE
1709
1710 if (val == RADEON_VCN_ENGINE_TYPE_DECODE) {
1711 decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[10];
1712
1713 if (decode_buffer->valid_buf_flag & 0x1)
1714 r = vcn_v4_0_dec_msg(p, ((u64)decode_buffer->msg_buffer_address_hi) << 32 |
1715 decode_buffer->msg_buffer_address_lo);
1716 }
1717 return r;
1718}
1719
bb4f196b 1720static const struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {
8da1170a
LL		 1721 .type = AMDGPU_RING_TYPE_VCN_ENC,
1722 .align_mask = 0x3f,
1723 .nop = VCN_ENC_CMD_NO_OP,
1724 .vmhub = AMDGPU_MMHUB_0,
bb4f196b
RD		 1725 .get_rptr = vcn_v4_0_unified_ring_get_rptr,
1726 .get_wptr = vcn_v4_0_unified_ring_get_wptr,
1727 .set_wptr = vcn_v4_0_unified_ring_set_wptr,
0b15205c 1728 .patch_cs_in_place = vcn_v4_0_ring_patch_cs_in_place,
8da1170a
LL		 1729 .emit_frame_size =
1730 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1731 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1732 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1733 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1734 1, /* vcn_v2_0_enc_ring_insert_end */
1735 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1736 .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1737 .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1738 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1739 .test_ring = amdgpu_vcn_enc_ring_test_ring,
bb4f196b 1740 .test_ib = amdgpu_vcn_unified_ring_test_ib,
8da1170a
LL		 1741 .insert_nop = amdgpu_ring_insert_nop,
1742 .insert_end = vcn_v2_0_enc_ring_insert_end,
1743 .pad_ib = amdgpu_ring_generic_pad_ib,
1744 .begin_use = amdgpu_vcn_ring_begin_use,
1745 .end_use = amdgpu_vcn_ring_end_use,
1746 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1747 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1748 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1749};
1750
1751/**
bb4f196b 1752 * vcn_v4_0_set_unified_ring_funcs - set unified ring functions
8da1170a
LL		 1753 *
1754 * @adev: amdgpu_device pointer
1755 *
bb4f196b 1756 * Set unified ring functions
8da1170a 1757 */
bb4f196b 1758static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev)
8da1170a
LL		 1759{
1760 int i;
1761
1762 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1763 if (adev->vcn.harvest_config & (1 << i))
1764 continue;
1765
bb4f196b
RD		 1766 adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_unified_ring_vm_funcs;
1767 adev->vcn.inst[i].ring_enc[0].me = i;
8da1170a 1768
bb4f196b 1769 DRM_INFO("VCN(%d) encode/decode are enabled in VM mode\n", i);
8da1170a
LL		 1770 }
1771}
1772
1773/**
1774 * vcn_v4_0_is_idle - check VCN block is idle
1775 *
1776 * @handle: amdgpu_device pointer
1777 *
1778 * Check whether VCN block is idle
1779 */
1780static bool vcn_v4_0_is_idle(void *handle)
1781{
1782 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1783 int i, ret = 1;
1784
1785 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1786 if (adev->vcn.harvest_config & (1 << i))
1787 continue;
1788
1789 ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
1790 }
1791
1792 return ret;
1793}
1794
1795/**
1796 * vcn_v4_0_wait_for_idle - wait for VCN block idle
1797 *
1798 * @handle: amdgpu_device pointer
1799 *
1800 * Wait for VCN block idle
1801 */
1802static int vcn_v4_0_wait_for_idle(void *handle)
1803{
1804 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1805 int i, ret = 0;
1806
1807 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1808 if (adev->vcn.harvest_config & (1 << i))
1809 continue;
1810
1811 ret = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE,
1812 UVD_STATUS__IDLE);
1813 if (ret)
1814 return ret;
1815 }
1816
1817 return ret;
1818}
1819
1820/**
1821 * vcn_v4_0_set_clockgating_state - set VCN block clockgating state
1822 *
1823 * @handle: amdgpu_device pointer
1824 * @state: clock gating state
1825 *
1826 * Set VCN block clockgating state
1827 */
1828static int vcn_v4_0_set_clockgating_state(void *handle, enum amd_clockgating_state state)
1829{
1830 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1831 bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
1832 int i;
1833
1834 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1835 if (adev->vcn.harvest_config & (1 << i))
1836 continue;
1837
1838 if (enable) {
1839 if (RREG32_SOC15(VCN, i, regUVD_STATUS) != UVD_STATUS__IDLE)
1840 return -EBUSY;
1841 vcn_v4_0_enable_clock_gating(adev, i);
1842 } else {
1843 vcn_v4_0_disable_clock_gating(adev, i);
1844 }
1845 }
1846
1847 return 0;
1848}
1849
1850/**
1851 * vcn_v4_0_set_powergating_state - set VCN block powergating state
1852 *
1853 * @handle: amdgpu_device pointer
1854 * @state: power gating state
1855 *
1856 * Set VCN block powergating state
1857 */
1858static int vcn_v4_0_set_powergating_state(void *handle, enum amd_powergating_state state)
1859{
1860 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1861 int ret;
1862
aa44beb5
JJ		 1863 /* for SRIOV, guest should not control VCN Power-gating
1864 * MMSCH FW should control Power-gating and clock-gating
1865 * guest should avoid touching CGC and PG
1866 */
1867 if (amdgpu_sriov_vf(adev)) {
1868 adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
1869 return 0;
1870 }
1871
8da1170a
LL		 1872 if(state == adev->vcn.cur_state)
1873 return 0;
1874
1875 if (state == AMD_PG_STATE_GATE)
1876 ret = vcn_v4_0_stop(adev);
1877 else
1878 ret = vcn_v4_0_start(adev);
1879
1880 if(!ret)
1881 adev->vcn.cur_state = state;
1882
1883 return ret;
1884}
1885
1886/**
1887 * vcn_v4_0_set_interrupt_state - set VCN block interrupt state
1888 *
1889 * @adev: amdgpu_device pointer
1890 * @source: interrupt sources
1891 * @type: interrupt types
1892 * @state: interrupt states
1893 *
1894 * Set VCN block interrupt state
1895 */
1896static int vcn_v4_0_set_interrupt_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1897 unsigned type, enum amdgpu_interrupt_state state)
1898{
1899 return 0;
1900}
1901
1902/**
1903 * vcn_v4_0_process_interrupt - process VCN block interrupt
1904 *
1905 * @adev: amdgpu_device pointer
1906 * @source: interrupt sources
1907 * @entry: interrupt entry from clients and sources
1908 *
1909 * Process VCN block interrupt
1910 */
1911static int vcn_v4_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1912 struct amdgpu_iv_entry *entry)
1913{
1914 uint32_t ip_instance;
1915
1916 switch (entry->client_id) {
1917 case SOC15_IH_CLIENTID_VCN:
1918 ip_instance = 0;
1919 break;
1920 case SOC15_IH_CLIENTID_VCN1:
1921 ip_instance = 1;
1922 break;
1923 default:
1924 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
1925 return 0;
1926 }
1927
1928 DRM_DEBUG("IH: VCN TRAP\n");
1929
1930 switch (entry->src_id) {
8da1170a
LL		 1931 case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1932 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
1933 break;
8da1170a
LL		 1934 default:
1935 DRM_ERROR("Unhandled interrupt: %d %d\n",
1936 entry->src_id, entry->src_data[0]);
1937 break;
1938 }
1939
1940 return 0;
1941}
1942
1943static const struct amdgpu_irq_src_funcs vcn_v4_0_irq_funcs = {
1944 .set = vcn_v4_0_set_interrupt_state,
1945 .process = vcn_v4_0_process_interrupt,
1946};
1947
1948/**
1949 * vcn_v4_0_set_irq_funcs - set VCN block interrupt irq functions
1950 *
1951 * @adev: amdgpu_device pointer
1952 *
1953 * Set VCN block interrupt irq functions
1954 */
1955static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev)
1956{
1957 int i;
1958
1959 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1960 if (adev->vcn.harvest_config & (1 << i))
1961 continue;
1962
1963 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
1964 adev->vcn.inst[i].irq.funcs = &vcn_v4_0_irq_funcs;
1965 }
1966}
1967
1968static const struct amd_ip_funcs vcn_v4_0_ip_funcs = {
1969 .name = "vcn_v4_0",
1970 .early_init = vcn_v4_0_early_init,
1971 .late_init = NULL,
1972 .sw_init = vcn_v4_0_sw_init,
1973 .sw_fini = vcn_v4_0_sw_fini,
1974 .hw_init = vcn_v4_0_hw_init,
1975 .hw_fini = vcn_v4_0_hw_fini,
1976 .suspend = vcn_v4_0_suspend,
1977 .resume = vcn_v4_0_resume,
1978 .is_idle = vcn_v4_0_is_idle,
1979 .wait_for_idle = vcn_v4_0_wait_for_idle,
1980 .check_soft_reset = NULL,
1981 .pre_soft_reset = NULL,
1982 .soft_reset = NULL,
1983 .post_soft_reset = NULL,
1984 .set_clockgating_state = vcn_v4_0_set_clockgating_state,
1985 .set_powergating_state = vcn_v4_0_set_powergating_state,
1986};
1987
1988const struct amdgpu_ip_block_version vcn_v4_0_ip_block =
1989{
1990 .type = AMD_IP_BLOCK_TYPE_VCN,
1991 .major = 4,
1992 .minor = 0,
1993 .rev = 0,
1994 .funcs = &vcn_v4_0_ip_funcs,
1995};
Linux 6.x block layer and io_uring tree(s)