[linux-block.git] / drivers / gpu / drm / amd / amdgpu / umsch_mm_v4_0.c

// SPDX-License-Identifier: MIT
/*
 * Copyright 2023 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include <linux/firmware.h>
#include <linux/module.h>
#include "amdgpu.h"
#include "soc15_common.h"
#include "soc21.h"
#include "vcn/vcn_4_0_0_offset.h"
#include "vcn/vcn_4_0_0_sh_mask.h"

#include "amdgpu_umsch_mm.h"
#include "umsch_mm_4_0_api_def.h"
#include "umsch_mm_v4_0.h"

#define WREG32_SOC15_UMSCH(ptr, reg, value) \
({	void *ret = ptr; 										\
	do { 												\
		uint32_t reg_offset = adev->reg_offset[VCN_HWIP][0][reg##_BASE_IDX] + reg; 		\
		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)					\
			ret = amdgpu_umsch_mm_add_cmd((&adev->umsch_mm), (ptr), (reg_offset), (value)); \
		else											\
			WREG32(reg_offset, value);							\
	} while (0);											\
	ret;												\
})

static int umsch_mm_v4_0_load_microcode(struct amdgpu_umsch_mm *umsch)
{
	struct amdgpu_device *adev = umsch->ring.adev;
	void* ptr = umsch->cmd_buf_ptr;
	uint32_t data;
	int r;

	r = amdgpu_umsch_mm_allocate_ucode_buffer(umsch);
	if (r)
		return r;

	r = amdgpu_umsch_mm_allocate_ucode_data_buffer(umsch);
	if (r)
		goto err_free_ucode_bo;

	data = RREG32_SOC15(VCN, 0, regUMSCH_MES_RESET_CTRL);
	data = REG_SET_FIELD(data, UMSCH_MES_RESET_CTRL, MES_CORE_SOFT_RESET, 0);
	ptr = WREG32_SOC15_UMSCH(ptr, regUMSCH_MES_RESET_CTRL, data);

	data = RREG32_SOC15(VCN, 0, regVCN_MES_CNTL);
	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_INVALIDATE_ICACHE, 1);
	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_PIPE0_RESET, 1);
	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_PIPE0_ACTIVE, 0);
	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_HALT, 1);
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_CNTL, data);

	data = RREG32_SOC15(VCN, 0, regVCN_MES_IC_BASE_CNTL);
	data = REG_SET_FIELD(data, VCN_MES_IC_BASE_CNTL, VMID, 0);
	data = REG_SET_FIELD(data, VCN_MES_IC_BASE_CNTL, EXE_DISABLE, 0);
	data = REG_SET_FIELD(data, VCN_MES_IC_BASE_CNTL, CACHE_POLICY, 0);
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_IC_BASE_CNTL, data);


	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_INTR_ROUTINE_START,
				 lower_32_bits(adev->umsch_mm.irq_start_addr >> 2));

	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_INTR_ROUTINE_START_HI,
				 upper_32_bits(adev->umsch_mm.irq_start_addr >> 2));

	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_PRGRM_CNTR_START,
				 lower_32_bits(adev->umsch_mm.uc_start_addr >> 2));
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_PRGRM_CNTR_START_HI,
				 upper_32_bits(adev->umsch_mm.uc_start_addr >> 2));

	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_INSTR_BASE_LO, 0);
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_INSTR_BASE_HI, 0);

	data = adev->umsch_mm.uc_start_addr + adev->umsch_mm.ucode_size - 1;
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_INSTR_MASK_LO, lower_32_bits(data));
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_INSTR_MASK_HI, upper_32_bits(data));

	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_IC_BASE_LO,
				 lower_32_bits(adev->umsch_mm.ucode_fw_gpu_addr));
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_IC_BASE_HI,
				 upper_32_bits(adev->umsch_mm.ucode_fw_gpu_addr));

	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_MIBOUND_LO, 0x1FFFFF);

	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_BASE0_LO,
				 lower_32_bits(adev->umsch_mm.data_start_addr));
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_BASE0_HI,
				 upper_32_bits(adev->umsch_mm.data_start_addr));

	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_MASK0_LO,
				 lower_32_bits(adev->umsch_mm.data_size - 1));
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_MASK0_HI,
				 upper_32_bits(adev->umsch_mm.data_size - 1));

	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_DC_BASE_LO,
				 lower_32_bits(adev->umsch_mm.data_fw_gpu_addr));
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_DC_BASE_HI,
				 upper_32_bits(adev->umsch_mm.data_fw_gpu_addr));

	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_MDBOUND_LO, 0x3FFFF);

	data = RREG32_SOC15(VCN, 0, regUVD_UMSCH_FORCE);
	data = REG_SET_FIELD(data, UVD_UMSCH_FORCE, IC_FORCE_GPUVM, 1);
	data = REG_SET_FIELD(data, UVD_UMSCH_FORCE, DC_FORCE_GPUVM, 1);
	ptr = WREG32_SOC15_UMSCH(ptr, regUVD_UMSCH_FORCE, data);

	data = RREG32_SOC15(VCN, 0, regVCN_MES_IC_OP_CNTL);
	data = REG_SET_FIELD(data, VCN_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
	data = REG_SET_FIELD(data, VCN_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_IC_OP_CNTL, data);

	data = RREG32_SOC15(VCN, 0, regVCN_MES_IC_OP_CNTL);
	data = REG_SET_FIELD(data, VCN_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_IC_OP_CNTL, data);

	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_GP0_LO, 0);
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_GP0_HI, 0);

	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_GP1_LO, 0);
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_GP1_HI, 0);

	data = RREG32_SOC15(VCN, 0, regVCN_MES_CNTL);
	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_INVALIDATE_ICACHE, 0);
	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_PIPE0_RESET, 0);
	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_HALT, 0);
	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_PIPE0_ACTIVE, 1);
	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_CNTL, data);

	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
		umsch_mm_psp_update_sram(adev,
					 (u32)((uintptr_t)ptr - (uintptr_t)umsch->cmd_buf_ptr));
	}

	r = SOC15_WAIT_ON_RREG(VCN, 0, regVCN_MES_MSTATUS_LO, 0xAAAAAAAA, 0xFFFFFFFF);
	if (r) {
		dev_err(adev->dev, "UMSCH FW Load: Failed, regVCN_MES_MSTATUS_LO: 0x%08x\n",
			RREG32_SOC15(VCN, 0, regVCN_MES_MSTATUS_LO));
		goto err_free_data_bo;
	}

	return 0;

err_free_data_bo:
	amdgpu_bo_free_kernel(&adev->umsch_mm.data_fw_obj,
			      &adev->umsch_mm.data_fw_gpu_addr,
			      (void **)&adev->umsch_mm.data_fw_ptr);
err_free_ucode_bo:
	amdgpu_bo_free_kernel(&adev->umsch_mm.ucode_fw_obj,
			      &adev->umsch_mm.ucode_fw_gpu_addr,
			      (void **)&adev->umsch_mm.ucode_fw_ptr);
	return r;
}

static void umsch_mm_v4_0_aggregated_doorbell_init(struct amdgpu_umsch_mm *umsch)
{
	struct amdgpu_device *adev = umsch->ring.adev;
	uint32_t data;

	data = RREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL0);
	data = REG_SET_FIELD(data, VCN_AGDB_CTRL0, OFFSET,
	       umsch->agdb_index[CONTEXT_PRIORITY_LEVEL_REALTIME]);
	data = REG_SET_FIELD(data, VCN_AGDB_CTRL0, EN, 1);
	WREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL0, data);

	data = RREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL1);
	data = REG_SET_FIELD(data, VCN_AGDB_CTRL1, OFFSET,
	       umsch->agdb_index[CONTEXT_PRIORITY_LEVEL_FOCUS]);
	data = REG_SET_FIELD(data, VCN_AGDB_CTRL1, EN, 1);
	WREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL1, data);

	data = RREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL2);
	data = REG_SET_FIELD(data, VCN_AGDB_CTRL2, OFFSET,
	       umsch->agdb_index[CONTEXT_PRIORITY_LEVEL_NORMAL]);
	data = REG_SET_FIELD(data, VCN_AGDB_CTRL2, EN, 1);
	WREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL2, data);

	data = RREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL3);
	data = REG_SET_FIELD(data, VCN_AGDB_CTRL3, OFFSET,
	       umsch->agdb_index[CONTEXT_PRIORITY_LEVEL_IDLE]);
	data = REG_SET_FIELD(data, VCN_AGDB_CTRL3, EN, 1);
	WREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL3, data);
}

static int umsch_mm_v4_0_ring_start(struct amdgpu_umsch_mm *umsch)
{
	struct amdgpu_ring *ring = &umsch->ring;
	struct amdgpu_device *adev = ring->adev;
	uint32_t data;

	data = RREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_DB_CTRL);
	data = REG_SET_FIELD(data, VCN_UMSCH_RB_DB_CTRL, OFFSET, ring->doorbell_index);
	data = REG_SET_FIELD(data, VCN_UMSCH_RB_DB_CTRL, EN, 1);
	WREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_DB_CTRL, data);

	adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell, ring->doorbell_index, 0);

	WREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_BASE_LO, lower_32_bits(ring->gpu_addr));
	WREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_BASE_HI, upper_32_bits(ring->gpu_addr));

	WREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_SIZE, ring->ring_size);

	data = RREG32_SOC15(VCN, 0, regVCN_RB_ENABLE);
	data &= ~(VCN_RB_ENABLE__AUDIO_RB_EN_MASK);
	WREG32_SOC15(VCN, 0, regVCN_RB_ENABLE, data);

	umsch_mm_v4_0_aggregated_doorbell_init(umsch);

	return 0;
}

static int umsch_mm_v4_0_ring_stop(struct amdgpu_umsch_mm *umsch)
{
	struct amdgpu_ring *ring = &umsch->ring;
	struct amdgpu_device *adev = ring->adev;
	uint32_t data;

	data = RREG32_SOC15(VCN, 0, regVCN_RB_ENABLE);
	data = REG_SET_FIELD(data, VCN_RB_ENABLE, UMSCH_RB_EN, 0);
	WREG32_SOC15(VCN, 0, regVCN_RB_ENABLE, data);

	data = RREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_DB_CTRL);
	data = REG_SET_FIELD(data, VCN_UMSCH_RB_DB_CTRL, EN, 0);
	WREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_DB_CTRL, data);

	return 0;
}

static int umsch_mm_v4_0_set_hw_resources(struct amdgpu_umsch_mm *umsch)
{
	union UMSCHAPI__SET_HW_RESOURCES set_hw_resources = {};
	struct amdgpu_device *adev = umsch->ring.adev;
	int r;

	set_hw_resources.header.type = UMSCH_API_TYPE_SCHEDULER;
	set_hw_resources.header.opcode = UMSCH_API_SET_HW_RSRC;
	set_hw_resources.header.dwsize = API_FRAME_SIZE_IN_DWORDS;

	set_hw_resources.vmid_mask_mm_vcn = umsch->vmid_mask_mm_vcn;
	set_hw_resources.vmid_mask_mm_vpe = umsch->vmid_mask_mm_vpe;
	set_hw_resources.engine_mask = umsch->engine_mask;

	set_hw_resources.vcn0_hqd_mask[0] = umsch->vcn0_hqd_mask;
	set_hw_resources.vcn1_hqd_mask[0] = umsch->vcn1_hqd_mask;
	set_hw_resources.vcn_hqd_mask[0] = umsch->vcn_hqd_mask[0];
	set_hw_resources.vcn_hqd_mask[1] = umsch->vcn_hqd_mask[1];
	set_hw_resources.vpe_hqd_mask[0] = umsch->vpe_hqd_mask;

	set_hw_resources.g_sch_ctx_gpu_mc_ptr = umsch->sch_ctx_gpu_addr;

	set_hw_resources.enable_level_process_quantum_check = 1;

	memcpy(set_hw_resources.mmhub_base, adev->reg_offset[MMHUB_HWIP][0],
	       sizeof(uint32_t) * 5);
	set_hw_resources.mmhub_version = amdgpu_ip_version(adev, MMHUB_HWIP, 0);

	memcpy(set_hw_resources.osssys_base, adev->reg_offset[OSSSYS_HWIP][0],
	       sizeof(uint32_t) * 5);
	set_hw_resources.osssys_version =
		amdgpu_ip_version(adev, OSSSYS_HWIP, 0);

	set_hw_resources.vcn_version = amdgpu_ip_version(adev, VCN_HWIP, 0);
	set_hw_resources.vpe_version = amdgpu_ip_version(adev, VPE_HWIP, 0);

	set_hw_resources.api_status.api_completion_fence_addr = umsch->ring.fence_drv.gpu_addr;
	set_hw_resources.api_status.api_completion_fence_value = ++umsch->ring.fence_drv.sync_seq;

	r = amdgpu_umsch_mm_submit_pkt(umsch, &set_hw_resources.max_dwords_in_api,
				       API_FRAME_SIZE_IN_DWORDS);
	if (r)
		return r;

	r = amdgpu_umsch_mm_query_fence(umsch);
	if (r) {
		dev_err(adev->dev, "UMSCH SET_HW_RESOURCES: Failed\n");
		return r;
	}

	return 0;
}

static int umsch_mm_v4_0_add_queue(struct amdgpu_umsch_mm *umsch,
				   struct umsch_mm_add_queue_input *input_ptr)
{
	struct amdgpu_device *adev = umsch->ring.adev;
	union UMSCHAPI__ADD_QUEUE add_queue = {};
	int r;

	add_queue.header.type = UMSCH_API_TYPE_SCHEDULER;
	add_queue.header.opcode = UMSCH_API_ADD_QUEUE;
	add_queue.header.dwsize = API_FRAME_SIZE_IN_DWORDS;

	add_queue.process_id = input_ptr->process_id;
	add_queue.page_table_base_addr = input_ptr->page_table_base_addr;
	add_queue.process_va_start = input_ptr->process_va_start;
	add_queue.process_va_end = input_ptr->process_va_end;
	add_queue.process_quantum = input_ptr->process_quantum;
	add_queue.process_csa_addr = input_ptr->process_csa_addr;
	add_queue.context_quantum = input_ptr->context_quantum;
	add_queue.context_csa_addr = input_ptr->context_csa_addr;
	add_queue.inprocess_context_priority = input_ptr->inprocess_context_priority;
	add_queue.context_global_priority_level =
		(enum UMSCH_AMD_PRIORITY_LEVEL)input_ptr->context_global_priority_level;
	add_queue.doorbell_offset_0 = input_ptr->doorbell_offset_0;
	add_queue.doorbell_offset_1 = input_ptr->doorbell_offset_1;
	add_queue.affinity.u32All = input_ptr->affinity;
	add_queue.mqd_addr = input_ptr->mqd_addr;
	add_queue.engine_type = (enum UMSCH_ENGINE_TYPE)input_ptr->engine_type;
	add_queue.h_context = input_ptr->h_context;
	add_queue.h_queue = input_ptr->h_queue;
	add_queue.vm_context_cntl = input_ptr->vm_context_cntl;
	add_queue.is_context_suspended = input_ptr->is_context_suspended;

	add_queue.api_status.api_completion_fence_addr = umsch->ring.fence_drv.gpu_addr;
	add_queue.api_status.api_completion_fence_value = ++umsch->ring.fence_drv.sync_seq;

	r = amdgpu_umsch_mm_submit_pkt(umsch, &add_queue.max_dwords_in_api,
				       API_FRAME_SIZE_IN_DWORDS);
	if (r)
		return r;

	r = amdgpu_umsch_mm_query_fence(umsch);
	if (r) {
		dev_err(adev->dev, "UMSCH ADD_QUEUE: Failed\n");
		return r;
	}

	return 0;
}

static int umsch_mm_v4_0_remove_queue(struct amdgpu_umsch_mm *umsch,
				      struct umsch_mm_remove_queue_input *input_ptr)
{
	union UMSCHAPI__REMOVE_QUEUE remove_queue = {};
	struct amdgpu_device *adev = umsch->ring.adev;
	int r;

	remove_queue.header.type = UMSCH_API_TYPE_SCHEDULER;
	remove_queue.header.opcode = UMSCH_API_REMOVE_QUEUE;
	remove_queue.header.dwsize = API_FRAME_SIZE_IN_DWORDS;

	remove_queue.doorbell_offset_0 = input_ptr->doorbell_offset_0;
	remove_queue.doorbell_offset_1 = input_ptr->doorbell_offset_1;
	remove_queue.context_csa_addr = input_ptr->context_csa_addr;

	remove_queue.api_status.api_completion_fence_addr = umsch->ring.fence_drv.gpu_addr;
	remove_queue.api_status.api_completion_fence_value = ++umsch->ring.fence_drv.sync_seq;

	r = amdgpu_umsch_mm_submit_pkt(umsch, &remove_queue.max_dwords_in_api,
				       API_FRAME_SIZE_IN_DWORDS);
	if (r)
		return r;

	r = amdgpu_umsch_mm_query_fence(umsch);
	if (r) {
		dev_err(adev->dev, "UMSCH REMOVE_QUEUE: Failed\n");
		return r;
	}

	return 0;
}

static int umsch_mm_v4_0_set_regs(struct amdgpu_umsch_mm *umsch)
{
	struct amdgpu_device *adev = container_of(umsch, struct amdgpu_device, umsch_mm);

	umsch->rb_wptr = SOC15_REG_OFFSET(VCN, 0, regVCN_UMSCH_RB_WPTR);
	umsch->rb_rptr = SOC15_REG_OFFSET(VCN, 0, regVCN_UMSCH_RB_RPTR);

	return 0;
}

static const struct umsch_mm_funcs umsch_mm_v4_0_funcs = {
	.set_hw_resources = umsch_mm_v4_0_set_hw_resources,
	.add_queue = umsch_mm_v4_0_add_queue,
	.remove_queue = umsch_mm_v4_0_remove_queue,
	.set_regs = umsch_mm_v4_0_set_regs,
	.init_microcode = amdgpu_umsch_mm_init_microcode,
	.load_microcode = umsch_mm_v4_0_load_microcode,
	.ring_init = amdgpu_umsch_mm_ring_init,
	.ring_start = umsch_mm_v4_0_ring_start,
	.ring_stop = umsch_mm_v4_0_ring_stop,
};

void umsch_mm_v4_0_set_funcs(struct amdgpu_umsch_mm *umsch)
{
	umsch->funcs = &umsch_mm_v4_0_funcs;
}
Commit	Line	Data
3488c79b LY	1	// SPDX-License-Identifier: MIT
	2	/*
	3	* Copyright 2023 Advanced Micro Devices, Inc.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the "Software"),
	7	* to deal in the Software without restriction, including without limitation
	8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	* and/or sell copies of the Software, and to permit persons to whom the
	10	* Software is furnished to do so, subject to the following conditions:
	11	*
	12	* The above copyright notice and this permission notice shall be included in
	13	* all copies or substantial portions of the Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	21	* OTHER DEALINGS IN THE SOFTWARE.
	22	*
	23	*/
	24
	25	#include <linux/firmware.h>
	26	#include <linux/module.h>
	27	#include "amdgpu.h"
	28	#include "soc15_common.h"
	29	#include "soc21.h"
	30	#include "vcn/vcn_4_0_0_offset.h"
	31	#include "vcn/vcn_4_0_0_sh_mask.h"
	32
	33	#include "amdgpu_umsch_mm.h"
	34	#include "umsch_mm_4_0_api_def.h"
	35	#include "umsch_mm_v4_0.h"
	36
4f949033 LY	37	#define WREG32_SOC15_UMSCH(ptr, reg, value) \
	38	({ void *ret = ptr; \
	39	do { \
	40	uint32_t reg_offset = adev->reg_offset[VCN_HWIP][0][reg##_BASE_IDX] + reg; \
	41	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) \
	42	ret = amdgpu_umsch_mm_add_cmd((&adev->umsch_mm), (ptr), (reg_offset), (value)); \
	43	else \
	44	WREG32(reg_offset, value); \
	45	} while (0); \
	46	ret; \
	47	})
	48
3488c79b LY	49	static int umsch_mm_v4_0_load_microcode(struct amdgpu_umsch_mm *umsch)
	50	{
	51	struct amdgpu_device *adev = umsch->ring.adev;
4f949033	52	void* ptr = umsch->cmd_buf_ptr;
3488c79b LY	53	uint32_t data;
	54	int r;
	55
	56	r = amdgpu_umsch_mm_allocate_ucode_buffer(umsch);
	57	if (r)
	58	return r;
	59
	60	r = amdgpu_umsch_mm_allocate_ucode_data_buffer(umsch);
	61	if (r)
	62	goto err_free_ucode_bo;
	63
	64	data = RREG32_SOC15(VCN, 0, regUMSCH_MES_RESET_CTRL);
	65	data = REG_SET_FIELD(data, UMSCH_MES_RESET_CTRL, MES_CORE_SOFT_RESET, 0);
4f949033	66	ptr = WREG32_SOC15_UMSCH(ptr, regUMSCH_MES_RESET_CTRL, data);
3488c79b LY	67
	68	data = RREG32_SOC15(VCN, 0, regVCN_MES_CNTL);
	69	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_INVALIDATE_ICACHE, 1);
	70	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_PIPE0_RESET, 1);
	71	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_PIPE0_ACTIVE, 0);
	72	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_HALT, 1);
4f949033	73	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_CNTL, data);
3488c79b LY	74
	75	data = RREG32_SOC15(VCN, 0, regVCN_MES_IC_BASE_CNTL);
	76	data = REG_SET_FIELD(data, VCN_MES_IC_BASE_CNTL, VMID, 0);
	77	data = REG_SET_FIELD(data, VCN_MES_IC_BASE_CNTL, EXE_DISABLE, 0);
	78	data = REG_SET_FIELD(data, VCN_MES_IC_BASE_CNTL, CACHE_POLICY, 0);
4f949033 LY	79	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_IC_BASE_CNTL, data);
	80
	81
	82	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_INTR_ROUTINE_START,
	83	lower_32_bits(adev->umsch_mm.irq_start_addr >> 2));
3488c79b	84
4f949033 LY	85	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_INTR_ROUTINE_START_HI,
4f949033 LY	86	upper_32_bits(adev->umsch_mm.irq_start_addr >> 2));
3488c79b	87
4f949033 LY	88	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_PRGRM_CNTR_START,
	89	lower_32_bits(adev->umsch_mm.uc_start_addr >> 2));
	90	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_PRGRM_CNTR_START_HI,
	91	upper_32_bits(adev->umsch_mm.uc_start_addr >> 2));
3488c79b	92
4f949033 LY	93	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_INSTR_BASE_LO, 0);
4f949033 LY	94	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_INSTR_BASE_HI, 0);
3488c79b LY	95
3488c79b LY	96	data = adev->umsch_mm.uc_start_addr + adev->umsch_mm.ucode_size - 1;
4f949033 LY	97	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_INSTR_MASK_LO, lower_32_bits(data));
4f949033 LY	98	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_INSTR_MASK_HI, upper_32_bits(data));
3488c79b	99
4f949033 LY	100	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_IC_BASE_LO,
	101	lower_32_bits(adev->umsch_mm.ucode_fw_gpu_addr));
	102	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_IC_BASE_HI,
	103	upper_32_bits(adev->umsch_mm.ucode_fw_gpu_addr));
3488c79b	104
4f949033	105	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_MIBOUND_LO, 0x1FFFFF);
3488c79b	106
4f949033 LY	107	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_BASE0_LO,
	108	lower_32_bits(adev->umsch_mm.data_start_addr));
	109	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_BASE0_HI,
	110	upper_32_bits(adev->umsch_mm.data_start_addr));
3488c79b	111
4f949033 LY	112	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_MASK0_LO,
	113	lower_32_bits(adev->umsch_mm.data_size - 1));
	114	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_LOCAL_MASK0_HI,
	115	upper_32_bits(adev->umsch_mm.data_size - 1));
3488c79b	116
4f949033 LY	117	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_DC_BASE_LO,
	118	lower_32_bits(adev->umsch_mm.data_fw_gpu_addr));
	119	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_DC_BASE_HI,
	120	upper_32_bits(adev->umsch_mm.data_fw_gpu_addr));
3488c79b	121
4f949033	122	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_MDBOUND_LO, 0x3FFFF);
3488c79b LY	123
	124	data = RREG32_SOC15(VCN, 0, regUVD_UMSCH_FORCE);
	125	data = REG_SET_FIELD(data, UVD_UMSCH_FORCE, IC_FORCE_GPUVM, 1);
	126	data = REG_SET_FIELD(data, UVD_UMSCH_FORCE, DC_FORCE_GPUVM, 1);
4f949033	127	ptr = WREG32_SOC15_UMSCH(ptr, regUVD_UMSCH_FORCE, data);
3488c79b LY	128
	129	data = RREG32_SOC15(VCN, 0, regVCN_MES_IC_OP_CNTL);
	130	data = REG_SET_FIELD(data, VCN_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
	131	data = REG_SET_FIELD(data, VCN_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
4f949033	132	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_IC_OP_CNTL, data);
3488c79b LY	133
	134	data = RREG32_SOC15(VCN, 0, regVCN_MES_IC_OP_CNTL);
	135	data = REG_SET_FIELD(data, VCN_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
4f949033	136	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_IC_OP_CNTL, data);
3488c79b	137
4f949033 LY	138	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_GP0_LO, 0);
4f949033 LY	139	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_GP0_HI, 0);
3488c79b	140
4f949033 LY	141	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_GP1_LO, 0);
4f949033 LY	142	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_GP1_HI, 0);
3488c79b LY	143
	144	data = RREG32_SOC15(VCN, 0, regVCN_MES_CNTL);
	145	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_INVALIDATE_ICACHE, 0);
	146	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_PIPE0_RESET, 0);
	147	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_HALT, 0);
	148	data = REG_SET_FIELD(data, VCN_MES_CNTL, MES_PIPE0_ACTIVE, 1);
4f949033 LY	149	ptr = WREG32_SOC15_UMSCH(ptr, regVCN_MES_CNTL, data);
	150
	151	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
	152	umsch_mm_psp_update_sram(adev,
	153	(u32)((uintptr_t)ptr - (uintptr_t)umsch->cmd_buf_ptr));
	154	}
3488c79b LY	155
	156	r = SOC15_WAIT_ON_RREG(VCN, 0, regVCN_MES_MSTATUS_LO, 0xAAAAAAAA, 0xFFFFFFFF);
	157	if (r) {
	158	dev_err(adev->dev, "UMSCH FW Load: Failed, regVCN_MES_MSTATUS_LO: 0x%08x\n",
	159	RREG32_SOC15(VCN, 0, regVCN_MES_MSTATUS_LO));
	160	goto err_free_data_bo;
	161	}
	162
	163	return 0;
	164
	165	err_free_data_bo:
	166	amdgpu_bo_free_kernel(&adev->umsch_mm.data_fw_obj,
	167	&adev->umsch_mm.data_fw_gpu_addr,
	168	(void **)&adev->umsch_mm.data_fw_ptr);
	169	err_free_ucode_bo:
	170	amdgpu_bo_free_kernel(&adev->umsch_mm.ucode_fw_obj,
	171	&adev->umsch_mm.ucode_fw_gpu_addr,
	172	(void **)&adev->umsch_mm.ucode_fw_ptr);
	173	return r;
	174	}
	175
	176	static void umsch_mm_v4_0_aggregated_doorbell_init(struct amdgpu_umsch_mm *umsch)
	177	{
	178	struct amdgpu_device *adev = umsch->ring.adev;
	179	uint32_t data;
	180
	181	data = RREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL0);
	182	data = REG_SET_FIELD(data, VCN_AGDB_CTRL0, OFFSET,
	183	umsch->agdb_index[CONTEXT_PRIORITY_LEVEL_REALTIME]);
	184	data = REG_SET_FIELD(data, VCN_AGDB_CTRL0, EN, 1);
	185	WREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL0, data);
	186
	187	data = RREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL1);
	188	data = REG_SET_FIELD(data, VCN_AGDB_CTRL1, OFFSET,
	189	umsch->agdb_index[CONTEXT_PRIORITY_LEVEL_FOCUS]);
	190	data = REG_SET_FIELD(data, VCN_AGDB_CTRL1, EN, 1);
	191	WREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL1, data);
	192
	193	data = RREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL2);
	194	data = REG_SET_FIELD(data, VCN_AGDB_CTRL2, OFFSET,
	195	umsch->agdb_index[CONTEXT_PRIORITY_LEVEL_NORMAL]);
	196	data = REG_SET_FIELD(data, VCN_AGDB_CTRL2, EN, 1);
	197	WREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL2, data);
	198
	199	data = RREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL3);
	200	data = REG_SET_FIELD(data, VCN_AGDB_CTRL3, OFFSET,
	201	umsch->agdb_index[CONTEXT_PRIORITY_LEVEL_IDLE]);
	202	data = REG_SET_FIELD(data, VCN_AGDB_CTRL3, EN, 1);
	203	WREG32_SOC15(VCN, 0, regVCN_AGDB_CTRL3, data);
	204	}
	205
	206	static int umsch_mm_v4_0_ring_start(struct amdgpu_umsch_mm *umsch)
	207	{
	208	struct amdgpu_ring *ring = &umsch->ring;
	209	struct amdgpu_device *adev = ring->adev;
	210	uint32_t data;
	211
	212	data = RREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_DB_CTRL);
	213	data = REG_SET_FIELD(data, VCN_UMSCH_RB_DB_CTRL, OFFSET, ring->doorbell_index);
	214	data = REG_SET_FIELD(data, VCN_UMSCH_RB_DB_CTRL, EN, 1);
	215	WREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_DB_CTRL, data);
	216
	217	adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell, ring->doorbell_index, 0);
	218
219	WREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_BASE_LO, lower_32_bits(ring->gpu_addr));
220	WREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
221
222	WREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_SIZE, ring->ring_size);
223
224	data = RREG32_SOC15(VCN, 0, regVCN_RB_ENABLE);
225	data &= ~(VCN_RB_ENABLE__AUDIO_RB_EN_MASK);
226	WREG32_SOC15(VCN, 0, regVCN_RB_ENABLE, data);
227
228	umsch_mm_v4_0_aggregated_doorbell_init(umsch);
229
230	return 0;
231	}
232
233	static int umsch_mm_v4_0_ring_stop(struct amdgpu_umsch_mm *umsch)
234	{
235	struct amdgpu_ring *ring = &umsch->ring;
236	struct amdgpu_device *adev = ring->adev;
237	uint32_t data;
238
239	data = RREG32_SOC15(VCN, 0, regVCN_RB_ENABLE);
240	data = REG_SET_FIELD(data, VCN_RB_ENABLE, UMSCH_RB_EN, 0);
241	WREG32_SOC15(VCN, 0, regVCN_RB_ENABLE, data);
242
243	data = RREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_DB_CTRL);
244	data = REG_SET_FIELD(data, VCN_UMSCH_RB_DB_CTRL, EN, 0);
245	WREG32_SOC15(VCN, 0, regVCN_UMSCH_RB_DB_CTRL, data);
246
247	return 0;
248	}
249
250	static int umsch_mm_v4_0_set_hw_resources(struct amdgpu_umsch_mm *umsch)
251	{
252	union UMSCHAPI__SET_HW_RESOURCES set_hw_resources = {};
253	struct amdgpu_device *adev = umsch->ring.adev;
254	int r;
255
256	set_hw_resources.header.type = UMSCH_API_TYPE_SCHEDULER;
257	set_hw_resources.header.opcode = UMSCH_API_SET_HW_RSRC;
258	set_hw_resources.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
259
260	set_hw_resources.vmid_mask_mm_vcn = umsch->vmid_mask_mm_vcn;
261	set_hw_resources.vmid_mask_mm_vpe = umsch->vmid_mask_mm_vpe;
262	set_hw_resources.engine_mask = umsch->engine_mask;
263
264	set_hw_resources.vcn0_hqd_mask[0] = umsch->vcn0_hqd_mask;
265	set_hw_resources.vcn1_hqd_mask[0] = umsch->vcn1_hqd_mask;
266	set_hw_resources.vcn_hqd_mask[0] = umsch->vcn_hqd_mask[0];
267	set_hw_resources.vcn_hqd_mask[1] = umsch->vcn_hqd_mask[1];
268	set_hw_resources.vpe_hqd_mask[0] = umsch->vpe_hqd_mask;
269
270	set_hw_resources.g_sch_ctx_gpu_mc_ptr = umsch->sch_ctx_gpu_addr;
271
272	set_hw_resources.enable_level_process_quantum_check = 1;
273
274	memcpy(set_hw_resources.mmhub_base, adev->reg_offset[MMHUB_HWIP][0],
275	sizeof(uint32_t) * 5);
4e8303cf	276	set_hw_resources.mmhub_version = amdgpu_ip_version(adev, MMHUB_HWIP, 0);
3488c79b LY	277
	278	memcpy(set_hw_resources.osssys_base, adev->reg_offset[OSSSYS_HWIP][0],
	279	sizeof(uint32_t) * 5);
4e8303cf LL	280	set_hw_resources.osssys_version =
4e8303cf LL	281	amdgpu_ip_version(adev, OSSSYS_HWIP, 0);
3488c79b	282
4e8303cf LL	283	set_hw_resources.vcn_version = amdgpu_ip_version(adev, VCN_HWIP, 0);
4e8303cf LL	284	set_hw_resources.vpe_version = amdgpu_ip_version(adev, VPE_HWIP, 0);
983ac45a	285
3488c79b LY	286	set_hw_resources.api_status.api_completion_fence_addr = umsch->ring.fence_drv.gpu_addr;
	287	set_hw_resources.api_status.api_completion_fence_value = ++umsch->ring.fence_drv.sync_seq;
	288
	289	r = amdgpu_umsch_mm_submit_pkt(umsch, &set_hw_resources.max_dwords_in_api,
	290	API_FRAME_SIZE_IN_DWORDS);
	291	if (r)
	292	return r;
	293
	294	r = amdgpu_umsch_mm_query_fence(umsch);
	295	if (r) {
	296	dev_err(adev->dev, "UMSCH SET_HW_RESOURCES: Failed\n");
	297	return r;
	298	}
	299
	300	return 0;
	301	}
	302
	303	static int umsch_mm_v4_0_add_queue(struct amdgpu_umsch_mm *umsch,
	304	struct umsch_mm_add_queue_input *input_ptr)
	305	{
	306	struct amdgpu_device *adev = umsch->ring.adev;
	307	union UMSCHAPI__ADD_QUEUE add_queue = {};
	308	int r;
	309
	310	add_queue.header.type = UMSCH_API_TYPE_SCHEDULER;
	311	add_queue.header.opcode = UMSCH_API_ADD_QUEUE;
	312	add_queue.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
	313
	314	add_queue.process_id = input_ptr->process_id;
	315	add_queue.page_table_base_addr = input_ptr->page_table_base_addr;
	316	add_queue.process_va_start = input_ptr->process_va_start;
	317	add_queue.process_va_end = input_ptr->process_va_end;
	318	add_queue.process_quantum = input_ptr->process_quantum;
	319	add_queue.process_csa_addr = input_ptr->process_csa_addr;
	320	add_queue.context_quantum = input_ptr->context_quantum;
	321	add_queue.context_csa_addr = input_ptr->context_csa_addr;
	322	add_queue.inprocess_context_priority = input_ptr->inprocess_context_priority;
	323	add_queue.context_global_priority_level =
	324	(enum UMSCH_AMD_PRIORITY_LEVEL)input_ptr->context_global_priority_level;
	325	add_queue.doorbell_offset_0 = input_ptr->doorbell_offset_0;
	326	add_queue.doorbell_offset_1 = input_ptr->doorbell_offset_1;
	327	add_queue.affinity.u32All = input_ptr->affinity;
	328	add_queue.mqd_addr = input_ptr->mqd_addr;
	329	add_queue.engine_type = (enum UMSCH_ENGINE_TYPE)input_ptr->engine_type;
	330	add_queue.h_context = input_ptr->h_context;
	331	add_queue.h_queue = input_ptr->h_queue;
	332	add_queue.vm_context_cntl = input_ptr->vm_context_cntl;
	333	add_queue.is_context_suspended = input_ptr->is_context_suspended;
	334
	335	add_queue.api_status.api_completion_fence_addr = umsch->ring.fence_drv.gpu_addr;
	336	add_queue.api_status.api_completion_fence_value = ++umsch->ring.fence_drv.sync_seq;
	337
	338	r = amdgpu_umsch_mm_submit_pkt(umsch, &add_queue.max_dwords_in_api,
	339	API_FRAME_SIZE_IN_DWORDS);
	340	if (r)
	341	return r;
	342
	343	r = amdgpu_umsch_mm_query_fence(umsch);
	344	if (r) {
	345	dev_err(adev->dev, "UMSCH ADD_QUEUE: Failed\n");
	346	return r;
	347	}
	348
	349	return 0;
350	}
351
352	static int umsch_mm_v4_0_remove_queue(struct amdgpu_umsch_mm *umsch,
353	struct umsch_mm_remove_queue_input *input_ptr)
354	{
355	union UMSCHAPI__REMOVE_QUEUE remove_queue = {};
356	struct amdgpu_device *adev = umsch->ring.adev;
357	int r;
358
359	remove_queue.header.type = UMSCH_API_TYPE_SCHEDULER;
360	remove_queue.header.opcode = UMSCH_API_REMOVE_QUEUE;
361	remove_queue.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
362
363	remove_queue.doorbell_offset_0 = input_ptr->doorbell_offset_0;
364	remove_queue.doorbell_offset_1 = input_ptr->doorbell_offset_1;
365	remove_queue.context_csa_addr = input_ptr->context_csa_addr;
366
367	remove_queue.api_status.api_completion_fence_addr = umsch->ring.fence_drv.gpu_addr;
368	remove_queue.api_status.api_completion_fence_value = ++umsch->ring.fence_drv.sync_seq;
369
370	r = amdgpu_umsch_mm_submit_pkt(umsch, &remove_queue.max_dwords_in_api,
371	API_FRAME_SIZE_IN_DWORDS);
372	if (r)
373	return r;
374
375	r = amdgpu_umsch_mm_query_fence(umsch);
376	if (r) {
377	dev_err(adev->dev, "UMSCH REMOVE_QUEUE: Failed\n");
378	return r;
379	}
380
381	return 0;
382	}
383
384	static int umsch_mm_v4_0_set_regs(struct amdgpu_umsch_mm *umsch)
385	{
386	struct amdgpu_device *adev = container_of(umsch, struct amdgpu_device, umsch_mm);
387
388	umsch->rb_wptr = SOC15_REG_OFFSET(VCN, 0, regVCN_UMSCH_RB_WPTR);
389	umsch->rb_rptr = SOC15_REG_OFFSET(VCN, 0, regVCN_UMSCH_RB_RPTR);
390
391	return 0;
392	}
393
394	static const struct umsch_mm_funcs umsch_mm_v4_0_funcs = {
395	.set_hw_resources = umsch_mm_v4_0_set_hw_resources,
396	.add_queue = umsch_mm_v4_0_add_queue,
397	.remove_queue = umsch_mm_v4_0_remove_queue,
398	.set_regs = umsch_mm_v4_0_set_regs,
399	.init_microcode = amdgpu_umsch_mm_init_microcode,
400	.load_microcode = umsch_mm_v4_0_load_microcode,
401	.ring_init = amdgpu_umsch_mm_ring_init,
402	.ring_start = umsch_mm_v4_0_ring_start,
403	.ring_stop = umsch_mm_v4_0_ring_stop,
404	};
405
406	void umsch_mm_v4_0_set_funcs(struct amdgpu_umsch_mm *umsch)
407	{
408	umsch->funcs = &umsch_mm_v4_0_funcs;
409	}