2 * Copyright 2023 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
25 #include "amdgpu_jpeg.h"
26 #include "amdgpu_pm.h"
29 #include "jpeg_v2_0.h"
30 #include "jpeg_v4_0_5.h"
31 #include "mmsch_v4_0.h"
33 #include "vcn/vcn_4_0_5_offset.h"
34 #include "vcn/vcn_4_0_5_sh_mask.h"
35 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
37 #define regUVD_JPEG_PITCH_INTERNAL_OFFSET 0x401f
39 static void jpeg_v4_0_5_set_dec_ring_funcs(struct amdgpu_device *adev);
40 static void jpeg_v4_0_5_set_irq_funcs(struct amdgpu_device *adev);
41 static int jpeg_v4_0_5_set_powergating_state(void *handle,
42 enum amd_powergating_state state);
44 static void jpeg_v4_0_5_dec_ring_set_wptr(struct amdgpu_ring *ring);
47 * jpeg_v4_0_5_early_init - set function pointers
49 * @handle: amdgpu_device pointer
51 * Set ring and irq function pointers
53 static int jpeg_v4_0_5_early_init(void *handle)
55 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
58 adev->jpeg.num_jpeg_inst = 1;
59 adev->jpeg.num_jpeg_rings = 1;
61 jpeg_v4_0_5_set_dec_ring_funcs(adev);
62 jpeg_v4_0_5_set_irq_funcs(adev);
68 * jpeg_v4_0_5_sw_init - sw init for JPEG block
70 * @handle: amdgpu_device pointer
72 * Load firmware and sw initialization
74 static int jpeg_v4_0_5_sw_init(void *handle)
76 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
77 struct amdgpu_ring *ring;
81 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
82 VCN_4_0__SRCID__JPEG_DECODE, &adev->jpeg.inst->irq);
86 /* JPEG DJPEG POISON EVENT */
87 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
88 VCN_4_0__SRCID_DJPEG0_POISON, &adev->jpeg.inst->irq);
92 /* JPEG EJPEG POISON EVENT */
93 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
94 VCN_4_0__SRCID_EJPEG0_POISON, &adev->jpeg.inst->irq);
98 r = amdgpu_jpeg_sw_init(adev);
102 r = amdgpu_jpeg_resume(adev);
106 ring = adev->jpeg.inst->ring_dec;
107 ring->use_doorbell = true;
108 ring->doorbell_index = amdgpu_sriov_vf(adev) ?
109 (((adev->doorbell_index.vcn.vcn_ring0_1) << 1) + 4) :
110 ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1);
111 ring->vm_hub = AMDGPU_MMHUB0(0);
113 sprintf(ring->name, "jpeg_dec");
114 r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq, 0,
115 AMDGPU_RING_PRIO_DEFAULT, NULL);
119 adev->jpeg.internal.jpeg_pitch[0] = regUVD_JPEG_PITCH_INTERNAL_OFFSET;
120 adev->jpeg.inst->external.jpeg_pitch[0] = SOC15_REG_OFFSET(JPEG, 0, regUVD_JPEG_PITCH);
126 * jpeg_v4_0_5_sw_fini - sw fini for JPEG block
128 * @handle: amdgpu_device pointer
130 * JPEG suspend and free up sw allocation
132 static int jpeg_v4_0_5_sw_fini(void *handle)
134 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
137 r = amdgpu_jpeg_suspend(adev);
141 r = amdgpu_jpeg_sw_fini(adev);
147 * jpeg_v4_0_5_hw_init - start and test JPEG block
149 * @handle: amdgpu_device pointer
152 static int jpeg_v4_0_5_hw_init(void *handle)
154 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
155 struct amdgpu_ring *ring = adev->jpeg.inst->ring_dec;
158 r = amdgpu_ring_test_helper(ring);
162 DRM_DEV_INFO(adev->dev, "JPEG decode initialized successfully.\n");
168 * jpeg_v4_0_5_hw_fini - stop the hardware block
170 * @handle: amdgpu_device pointer
172 * Stop the JPEG block, mark ring as not ready any more
174 static int jpeg_v4_0_5_hw_fini(void *handle)
176 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
178 cancel_delayed_work_sync(&adev->vcn.idle_work);
179 if (!amdgpu_sriov_vf(adev)) {
180 if (adev->jpeg.cur_state != AMD_PG_STATE_GATE &&
181 RREG32_SOC15(JPEG, 0, regUVD_JRBC_STATUS))
182 jpeg_v4_0_5_set_powergating_state(adev, AMD_PG_STATE_GATE);
184 amdgpu_irq_put(adev, &adev->jpeg.inst->irq, 0);
190 * jpeg_v4_0_5_suspend - suspend JPEG block
192 * @handle: amdgpu_device pointer
194 * HW fini and suspend JPEG block
196 static int jpeg_v4_0_5_suspend(void *handle)
198 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
201 r = jpeg_v4_0_5_hw_fini(adev);
205 r = amdgpu_jpeg_suspend(adev);
211 * jpeg_v4_0_5_resume - resume JPEG block
213 * @handle: amdgpu_device pointer
215 * Resume firmware and hw init JPEG block
217 static int jpeg_v4_0_5_resume(void *handle)
219 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
222 r = amdgpu_jpeg_resume(adev);
226 r = jpeg_v4_0_5_hw_init(adev);
231 static void jpeg_v4_0_5_disable_clock_gating(struct amdgpu_device *adev)
235 data = RREG32_SOC15(JPEG, 0, regJPEG_CGC_CTRL);
236 if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
237 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
238 data &= (~JPEG_CGC_CTRL__JPEG_DEC_MODE_MASK);
240 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
243 data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
244 data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
245 WREG32_SOC15(JPEG, 0, regJPEG_CGC_CTRL, data);
247 data = RREG32_SOC15(JPEG, 0, regJPEG_CGC_GATE);
248 data &= ~(JPEG_CGC_GATE__JPEG_DEC_MASK
249 | JPEG_CGC_GATE__JPEG2_DEC_MASK
250 | JPEG_CGC_GATE__JMCIF_MASK
251 | JPEG_CGC_GATE__JRBBM_MASK);
252 WREG32_SOC15(JPEG, 0, regJPEG_CGC_GATE, data);
255 static void jpeg_v4_0_5_enable_clock_gating(struct amdgpu_device *adev)
259 data = RREG32_SOC15(JPEG, 0, regJPEG_CGC_CTRL);
260 if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
261 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
262 data |= JPEG_CGC_CTRL__JPEG_DEC_MODE_MASK;
264 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
267 data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
268 data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
269 WREG32_SOC15(JPEG, 0, regJPEG_CGC_CTRL, data);
271 data = RREG32_SOC15(JPEG, 0, regJPEG_CGC_GATE);
272 data |= (JPEG_CGC_GATE__JPEG_DEC_MASK
273 |JPEG_CGC_GATE__JPEG2_DEC_MASK
274 |JPEG_CGC_GATE__JMCIF_MASK
275 |JPEG_CGC_GATE__JRBBM_MASK);
276 WREG32_SOC15(JPEG, 0, regJPEG_CGC_GATE, data);
279 static int jpeg_v4_0_5_disable_static_power_gating(struct amdgpu_device *adev)
281 if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
282 WREG32(SOC15_REG_OFFSET(JPEG, 0, regUVD_IPX_DLDO_CONFIG),
283 1 << UVD_IPX_DLDO_CONFIG__ONO1_PWR_CONFIG__SHIFT);
284 SOC15_WAIT_ON_RREG(JPEG, 0, regUVD_IPX_DLDO_STATUS,
285 0, UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS_MASK);
288 /* disable anti hang mechanism */
289 WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regUVD_JPEG_POWER_STATUS), 0,
290 ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
292 /* keep the JPEG in static PG mode */
293 WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regUVD_JPEG_POWER_STATUS), 0,
294 ~UVD_JPEG_POWER_STATUS__JPEG_PG_MODE_MASK);
299 static int jpeg_v4_0_5_enable_static_power_gating(struct amdgpu_device *adev)
301 /* enable anti hang mechanism */
302 WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regUVD_JPEG_POWER_STATUS),
303 UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK,
304 ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
306 if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
307 WREG32(SOC15_REG_OFFSET(JPEG, 0, regUVD_IPX_DLDO_CONFIG),
308 2 << UVD_IPX_DLDO_CONFIG__ONO1_PWR_CONFIG__SHIFT);
309 SOC15_WAIT_ON_RREG(JPEG, 0, regUVD_IPX_DLDO_STATUS,
310 1 << UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS__SHIFT,
311 UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS_MASK);
318 * jpeg_v4_0_5_start - start JPEG block
320 * @adev: amdgpu_device pointer
322 * Setup and start the JPEG block
324 static int jpeg_v4_0_5_start(struct amdgpu_device *adev)
326 struct amdgpu_ring *ring = adev->jpeg.inst->ring_dec;
329 if (adev->pm.dpm_enabled)
330 amdgpu_dpm_enable_jpeg(adev, true);
332 /* doorbell programming is done for every playback */
333 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
334 (adev->doorbell_index.vcn.vcn_ring0_1 << 1), 0);
336 WREG32_SOC15(VCN, 0, regVCN_JPEG_DB_CTRL,
337 ring->doorbell_index << VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
338 VCN_JPEG_DB_CTRL__EN_MASK);
340 /* disable power gating */
341 r = jpeg_v4_0_5_disable_static_power_gating(adev);
345 /* JPEG disable CGC */
346 jpeg_v4_0_5_disable_clock_gating(adev);
348 /* MJPEG global tiling registers */
349 WREG32_SOC15(JPEG, 0, regJPEG_DEC_GFX10_ADDR_CONFIG,
350 adev->gfx.config.gb_addr_config);
353 /* enable JMI channel */
354 WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI_CNTL), 0,
355 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
357 /* enable System Interrupt for JRBC */
358 WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regJPEG_SYS_INT_EN),
359 JPEG_SYS_INT_EN__DJRBC_MASK,
360 ~JPEG_SYS_INT_EN__DJRBC_MASK);
362 WREG32_SOC15(JPEG, 0, regUVD_LMI_JRBC_RB_VMID, 0);
363 WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_CNTL, (0x00000001L | 0x00000002L));
364 WREG32_SOC15(JPEG, 0, regUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
365 lower_32_bits(ring->gpu_addr));
366 WREG32_SOC15(JPEG, 0, regUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
367 upper_32_bits(ring->gpu_addr));
368 WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_RPTR, 0);
369 WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_WPTR, 0);
370 WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_CNTL, 0x00000002L);
371 WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_SIZE, ring->ring_size / 4);
372 ring->wptr = RREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_WPTR);
378 * jpeg_v4_0_5_stop - stop JPEG block
380 * @adev: amdgpu_device pointer
382 * stop the JPEG block
384 static int jpeg_v4_0_5_stop(struct amdgpu_device *adev)
389 WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI_CNTL),
390 UVD_JMI_CNTL__SOFT_RESET_MASK,
391 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
393 jpeg_v4_0_5_enable_clock_gating(adev);
395 /* enable power gating */
396 r = jpeg_v4_0_5_enable_static_power_gating(adev);
400 if (adev->pm.dpm_enabled)
401 amdgpu_dpm_enable_jpeg(adev, false);
407 * jpeg_v4_0_5_dec_ring_get_rptr - get read pointer
409 * @ring: amdgpu_ring pointer
411 * Returns the current hardware read pointer
413 static uint64_t jpeg_v4_0_5_dec_ring_get_rptr(struct amdgpu_ring *ring)
415 struct amdgpu_device *adev = ring->adev;
417 return RREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_RPTR);
421 * jpeg_v4_0_5_dec_ring_get_wptr - get write pointer
423 * @ring: amdgpu_ring pointer
425 * Returns the current hardware write pointer
427 static uint64_t jpeg_v4_0_5_dec_ring_get_wptr(struct amdgpu_ring *ring)
429 struct amdgpu_device *adev = ring->adev;
431 if (ring->use_doorbell)
432 return *ring->wptr_cpu_addr;
434 return RREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_WPTR);
438 * jpeg_v4_0_5_dec_ring_set_wptr - set write pointer
440 * @ring: amdgpu_ring pointer
442 * Commits the write pointer to the hardware
444 static void jpeg_v4_0_5_dec_ring_set_wptr(struct amdgpu_ring *ring)
446 struct amdgpu_device *adev = ring->adev;
448 if (ring->use_doorbell) {
449 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
450 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
452 WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_WPTR, lower_32_bits(ring->wptr));
456 static bool jpeg_v4_0_5_is_idle(void *handle)
458 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
461 ret &= (((RREG32_SOC15(JPEG, 0, regUVD_JRBC_STATUS) &
462 UVD_JRBC_STATUS__RB_JOB_DONE_MASK) ==
463 UVD_JRBC_STATUS__RB_JOB_DONE_MASK));
468 static int jpeg_v4_0_5_wait_for_idle(void *handle)
470 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
472 return SOC15_WAIT_ON_RREG(JPEG, 0, regUVD_JRBC_STATUS,
473 UVD_JRBC_STATUS__RB_JOB_DONE_MASK,
474 UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
477 static int jpeg_v4_0_5_set_clockgating_state(void *handle,
478 enum amd_clockgating_state state)
480 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
481 bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
484 if (!jpeg_v4_0_5_is_idle(handle))
486 jpeg_v4_0_5_enable_clock_gating(adev);
488 jpeg_v4_0_5_disable_clock_gating(adev);
494 static int jpeg_v4_0_5_set_powergating_state(void *handle,
495 enum amd_powergating_state state)
497 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
500 if (amdgpu_sriov_vf(adev)) {
501 adev->jpeg.cur_state = AMD_PG_STATE_UNGATE;
505 if (state == adev->jpeg.cur_state)
508 if (state == AMD_PG_STATE_GATE)
509 ret = jpeg_v4_0_5_stop(adev);
511 ret = jpeg_v4_0_5_start(adev);
514 adev->jpeg.cur_state = state;
519 static int jpeg_v4_0_5_set_interrupt_state(struct amdgpu_device *adev,
520 struct amdgpu_irq_src *source,
522 enum amdgpu_interrupt_state state)
527 static int jpeg_v4_0_5_process_interrupt(struct amdgpu_device *adev,
528 struct amdgpu_irq_src *source,
529 struct amdgpu_iv_entry *entry)
531 DRM_DEBUG("IH: JPEG TRAP\n");
533 switch (entry->src_id) {
534 case VCN_4_0__SRCID__JPEG_DECODE:
535 amdgpu_fence_process(adev->jpeg.inst->ring_dec);
537 case VCN_4_0__SRCID_DJPEG0_POISON:
538 case VCN_4_0__SRCID_EJPEG0_POISON:
539 amdgpu_jpeg_process_poison_irq(adev, source, entry);
542 DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
543 entry->src_id, entry->src_data[0]);
550 static const struct amd_ip_funcs jpeg_v4_0_5_ip_funcs = {
551 .name = "jpeg_v4_0_5",
552 .early_init = jpeg_v4_0_5_early_init,
554 .sw_init = jpeg_v4_0_5_sw_init,
555 .sw_fini = jpeg_v4_0_5_sw_fini,
556 .hw_init = jpeg_v4_0_5_hw_init,
557 .hw_fini = jpeg_v4_0_5_hw_fini,
558 .suspend = jpeg_v4_0_5_suspend,
559 .resume = jpeg_v4_0_5_resume,
560 .is_idle = jpeg_v4_0_5_is_idle,
561 .wait_for_idle = jpeg_v4_0_5_wait_for_idle,
562 .check_soft_reset = NULL,
563 .pre_soft_reset = NULL,
565 .post_soft_reset = NULL,
566 .set_clockgating_state = jpeg_v4_0_5_set_clockgating_state,
567 .set_powergating_state = jpeg_v4_0_5_set_powergating_state,
570 static const struct amdgpu_ring_funcs jpeg_v4_0_5_dec_ring_vm_funcs = {
571 .type = AMDGPU_RING_TYPE_VCN_JPEG,
573 .get_rptr = jpeg_v4_0_5_dec_ring_get_rptr,
574 .get_wptr = jpeg_v4_0_5_dec_ring_get_wptr,
575 .set_wptr = jpeg_v4_0_5_dec_ring_set_wptr,
577 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
578 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
579 8 + /* jpeg_v4_0_5_dec_ring_emit_vm_flush */
580 18 + 18 + /* jpeg_v4_0_5_dec_ring_emit_fence x2 vm fence */
582 .emit_ib_size = 22, /* jpeg_v4_0_5_dec_ring_emit_ib */
583 .emit_ib = jpeg_v2_0_dec_ring_emit_ib,
584 .emit_fence = jpeg_v2_0_dec_ring_emit_fence,
585 .emit_vm_flush = jpeg_v2_0_dec_ring_emit_vm_flush,
586 .test_ring = amdgpu_jpeg_dec_ring_test_ring,
587 .test_ib = amdgpu_jpeg_dec_ring_test_ib,
588 .insert_nop = jpeg_v2_0_dec_ring_nop,
589 .insert_start = jpeg_v2_0_dec_ring_insert_start,
590 .insert_end = jpeg_v2_0_dec_ring_insert_end,
591 .pad_ib = amdgpu_ring_generic_pad_ib,
592 .begin_use = amdgpu_jpeg_ring_begin_use,
593 .end_use = amdgpu_jpeg_ring_end_use,
594 .emit_wreg = jpeg_v2_0_dec_ring_emit_wreg,
595 .emit_reg_wait = jpeg_v2_0_dec_ring_emit_reg_wait,
596 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
599 static void jpeg_v4_0_5_set_dec_ring_funcs(struct amdgpu_device *adev)
601 adev->jpeg.inst->ring_dec->funcs = &jpeg_v4_0_5_dec_ring_vm_funcs;
602 DRM_DEV_INFO(adev->dev, "JPEG decode is enabled in VM mode\n");
605 static const struct amdgpu_irq_src_funcs jpeg_v4_0_5_irq_funcs = {
606 .set = jpeg_v4_0_5_set_interrupt_state,
607 .process = jpeg_v4_0_5_process_interrupt,
610 static void jpeg_v4_0_5_set_irq_funcs(struct amdgpu_device *adev)
612 adev->jpeg.inst->irq.num_types = 1;
613 adev->jpeg.inst->irq.funcs = &jpeg_v4_0_5_irq_funcs;
616 const struct amdgpu_ip_block_version jpeg_v4_0_5_ip_block = {
617 .type = AMD_IP_BLOCK_TYPE_JPEG,
621 .funcs = &jpeg_v4_0_5_ip_funcs,