[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / vega10_ih.c

/*
 * Copyright 2016 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */
#include <drm/drmP.h>
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "soc15.h"

#include "oss/osssys_4_0_offset.h"
#include "oss/osssys_4_0_sh_mask.h"

#include "soc15_common.h"
#include "vega10_ih.h"


static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);

/**
 * vega10_ih_enable_interrupts - Enable the interrupt ring buffer
 *
 * @adev: amdgpu_device pointer
 *
 * Enable the interrupt ring buffer (VEGA10).
 */
static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
{
	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);

	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
	adev->irq.ih.enabled = true;
}

/**
 * vega10_ih_disable_interrupts - Disable the interrupt ring buffer
 *
 * @adev: amdgpu_device pointer
 *
 * Disable the interrupt ring buffer (VEGA10).
 */
static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
{
	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);

	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
	/* set rptr, wptr to 0 */
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
	adev->irq.ih.enabled = false;
	adev->irq.ih.rptr = 0;
}

/**
 * vega10_ih_irq_init - init and enable the interrupt ring
 *
 * @adev: amdgpu_device pointer
 *
 * Allocate a ring buffer for the interrupt controller,
 * enable the RLC, disable interrupts, enable the IH
 * ring buffer and enable it (VI).
 * Called at device load and reume.
 * Returns 0 for success, errors for failure.
 */
static int vega10_ih_irq_init(struct amdgpu_device *adev)
{
	int ret = 0;
	int rb_bufsz;
	u32 ih_rb_cntl, ih_doorbell_rtpr;
	u32 tmp;
	u64 wptr_off;

	/* disable irqs */
	vega10_ih_disable_interrupts(adev);

	adev->nbio_funcs->ih_control(adev);

	ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
	if (adev->irq.ih.use_bus_addr) {
		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8);
		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
	} else {
		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (adev->irq.ih.gpu_addr >> 40) & 0xff);
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
	}
	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);

	if (adev->irq.msi_enabled)
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);

	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);

	/* set the writeback address whether it's enabled or not */
	if (adev->irq.ih.use_bus_addr)
		wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
	else
		wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off));
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFFFF);

	/* set rptr, wptr to 0 */
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);

	ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
	if (adev->irq.ih.use_doorbell) {
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 OFFSET, adev->irq.ih.doorbell_index);
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 ENABLE, 1);
	} else {
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 ENABLE, 0);
	}
	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
	adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell,
					    adev->irq.ih.doorbell_index);

	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
			    CLIENT18_IS_STORM_CLIENT, 1);
	WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);

	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
	WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);

	pci_set_master(adev->pdev);

	/* enable interrupts */
	vega10_ih_enable_interrupts(adev);

	return ret;
}

/**
 * vega10_ih_irq_disable - disable interrupts
 *
 * @adev: amdgpu_device pointer
 *
 * Disable interrupts on the hw (VEGA10).
 */
static void vega10_ih_irq_disable(struct amdgpu_device *adev)
{
	vega10_ih_disable_interrupts(adev);

	/* Wait and acknowledge irq */
	mdelay(1);
}

/**
 * vega10_ih_get_wptr - get the IH ring buffer wptr
 *
 * @adev: amdgpu_device pointer
 *
 * Get the IH ring buffer wptr from either the register
 * or the writeback memory buffer (VEGA10).  Also check for
 * ring buffer overflow and deal with it.
 * Returns the value of the wptr.
 */
static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
{
	u32 wptr, tmp;

	if (adev->irq.ih.use_bus_addr)
		wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);
	else
		wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);

	if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
		wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);

		/* When a ring buffer overflow happen start parsing interrupt
		 * from the last not overwritten vector (wptr + 32). Hopefully
		 * this should allow us to catchup.
		 */
		tmp = (wptr + 32) & adev->irq.ih.ptr_mask;
		dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
			wptr, adev->irq.ih.rptr, tmp);
		adev->irq.ih.rptr = tmp;

		tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
		WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
	}
	return (wptr & adev->irq.ih.ptr_mask);
}

/**
 * vega10_ih_decode_iv - decode an interrupt vector
 *
 * @adev: amdgpu_device pointer
 *
 * Decodes the interrupt vector at the current rptr
 * position and also advance the position.
 */
static void vega10_ih_decode_iv(struct amdgpu_device *adev,
				 struct amdgpu_iv_entry *entry)
{
	/* wptr/rptr are in bytes! */
	u32 ring_index = adev->irq.ih.rptr >> 2;
	uint32_t dw[8];

	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);
	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);

	entry->client_id = dw[0] & 0xff;
	entry->src_id = (dw[0] >> 8) & 0xff;
	entry->ring_id = (dw[0] >> 16) & 0xff;
	entry->vmid = (dw[0] >> 24) & 0xf;
	entry->vmid_src = (dw[0] >> 31);
	entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
	entry->timestamp_src = dw[2] >> 31;
	entry->pasid = dw[3] & 0xffff;
	entry->pasid_src = dw[3] >> 31;
	entry->src_data[0] = dw[4];
	entry->src_data[1] = dw[5];
	entry->src_data[2] = dw[6];
	entry->src_data[3] = dw[7];


	/* wptr/rptr are in bytes! */
	adev->irq.ih.rptr += 32;
}

/**
 * vega10_ih_set_rptr - set the IH ring buffer rptr
 *
 * @adev: amdgpu_device pointer
 *
 * Set the IH ring buffer rptr.
 */
static void vega10_ih_set_rptr(struct amdgpu_device *adev)
{
	if (adev->irq.ih.use_doorbell) {
		/* XXX check if swapping is necessary on BE */
		if (adev->irq.ih.use_bus_addr)
			adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
		else
			adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
		WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
	} else {
		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, adev->irq.ih.rptr);
	}
}

static int vega10_ih_early_init(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	vega10_ih_set_interrupt_funcs(adev);
	return 0;
}

static int vega10_ih_sw_init(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true);
	if (r)
		return r;

	adev->irq.ih.use_doorbell = true;
	adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;

	r = amdgpu_irq_init(adev);

	return r;
}

static int vega10_ih_sw_fini(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	amdgpu_irq_fini(adev);
	amdgpu_ih_ring_fini(adev, &adev->irq.ih);

	return 0;
}

static int vega10_ih_hw_init(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = vega10_ih_irq_init(adev);
	if (r)
		return r;

	return 0;
}

static int vega10_ih_hw_fini(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	vega10_ih_irq_disable(adev);

	return 0;
}

static int vega10_ih_suspend(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	return vega10_ih_hw_fini(adev);
}

static int vega10_ih_resume(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	return vega10_ih_hw_init(adev);
}

static bool vega10_ih_is_idle(void *handle)
{
	/* todo */
	return true;
}

static int vega10_ih_wait_for_idle(void *handle)
{
	/* todo */
	return -ETIMEDOUT;
}

static int vega10_ih_soft_reset(void *handle)
{
	/* todo */

	return 0;
}

static int vega10_ih_set_clockgating_state(void *handle,
					  enum amd_clockgating_state state)
{
	return 0;
}

static int vega10_ih_set_powergating_state(void *handle,
					  enum amd_powergating_state state)
{
	return 0;
}

const struct amd_ip_funcs vega10_ih_ip_funcs = {
	.name = "vega10_ih",
	.early_init = vega10_ih_early_init,
	.late_init = NULL,
	.sw_init = vega10_ih_sw_init,
	.sw_fini = vega10_ih_sw_fini,
	.hw_init = vega10_ih_hw_init,
	.hw_fini = vega10_ih_hw_fini,
	.suspend = vega10_ih_suspend,
	.resume = vega10_ih_resume,
	.is_idle = vega10_ih_is_idle,
	.wait_for_idle = vega10_ih_wait_for_idle,
	.soft_reset = vega10_ih_soft_reset,
	.set_clockgating_state = vega10_ih_set_clockgating_state,
	.set_powergating_state = vega10_ih_set_powergating_state,
};

static const struct amdgpu_ih_funcs vega10_ih_funcs = {
	.get_wptr = vega10_ih_get_wptr,
	.decode_iv = vega10_ih_decode_iv,
	.set_rptr = vega10_ih_set_rptr
};

static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
{
	adev->irq.ih_funcs = &vega10_ih_funcs;
}

const struct amdgpu_ip_block_version vega10_ih_ip_block =
{
	.type = AMD_IP_BLOCK_TYPE_IH,
	.major = 4,
	.minor = 0,
	.rev = 0,
	.funcs = &vega10_ih_ip_funcs,
};
Commit	Line	Data
282aae55 KW	1	/*
	2	* Copyright 2016 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*
	22	*/
248a1d6f	23	#include <drm/drmP.h>
282aae55 KW	24	#include "amdgpu.h"
	25	#include "amdgpu_ih.h"
	26	#include "soc15.h"
	27
8af7454e FX	28	#include "oss/osssys_4_0_offset.h"
8af7454e FX	29	#include "oss/osssys_4_0_sh_mask.h"
282aae55 KW	30
	31	#include "soc15_common.h"
	32	#include "vega10_ih.h"
	33
	34
	35
	36	static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
	37
	38	/**
	39	* vega10_ih_enable_interrupts - Enable the interrupt ring buffer
	40	*
	41	* @adev: amdgpu_device pointer
	42	*
	43	* Enable the interrupt ring buffer (VEGA10).
	44	*/
	45	static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
	46	{
b2b7e457	47	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
282aae55 KW	48
	49	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
	50	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
b2b7e457	51	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
282aae55 KW	52	adev->irq.ih.enabled = true;
	53	}
	54
	55	/**
	56	* vega10_ih_disable_interrupts - Disable the interrupt ring buffer
	57	*
	58	* @adev: amdgpu_device pointer
	59	*
	60	* Disable the interrupt ring buffer (VEGA10).
	61	*/
	62	static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
	63	{
b2b7e457	64	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
282aae55 KW	65
	66	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
	67	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
b2b7e457	68	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
282aae55	69	/* set rptr, wptr to 0 */
b2b7e457 HZ	70	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
b2b7e457 HZ	71	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
282aae55 KW	72	adev->irq.ih.enabled = false;
	73	adev->irq.ih.rptr = 0;
	74	}
	75
	76	/**
	77	* vega10_ih_irq_init - init and enable the interrupt ring
	78	*
	79	* @adev: amdgpu_device pointer
	80	*
	81	* Allocate a ring buffer for the interrupt controller,
	82	* enable the RLC, disable interrupts, enable the IH
	83	* ring buffer and enable it (VI).
	84	* Called at device load and reume.
	85	* Returns 0 for success, errors for failure.
	86	*/
	87	static int vega10_ih_irq_init(struct amdgpu_device *adev)
	88	{
	89	int ret = 0;
	90	int rb_bufsz;
	91	u32 ih_rb_cntl, ih_doorbell_rtpr;
	92	u32 tmp;
	93	u64 wptr_off;
	94
	95	/* disable irqs */
	96	vega10_ih_disable_interrupts(adev);
	97
bf383fb6	98	adev->nbio_funcs->ih_control(adev);
282aae55	99
b2b7e457	100	ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
282aae55 KW	101	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
282aae55 KW	102	if (adev->irq.ih.use_bus_addr) {
b2b7e457 HZ	103	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8);
b2b7e457 HZ	104	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
282aae55 KW	105	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
282aae55 KW	106	} else {
b2b7e457 HZ	107	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
b2b7e457 HZ	108	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (adev->irq.ih.gpu_addr >> 40) & 0xff);
282aae55 KW	109	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
	110	}
	111	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
	112	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
	113	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
	114	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
	115	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
	116	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
	117	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
	118	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
	119	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
	120
	121	if (adev->irq.msi_enabled)
	122	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);
	123
b2b7e457	124	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
282aae55 KW	125
	126	/* set the writeback address whether it's enabled or not */
	127	if (adev->irq.ih.use_bus_addr)
	128	wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
	129	else
	130	wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
b2b7e457	131	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off));
c837243f	132	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFFFF);
282aae55 KW	133
282aae55 KW	134	/* set rptr, wptr to 0 */
b2b7e457 HZ	135	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
b2b7e457 HZ	136	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
282aae55	137
b2b7e457	138	ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
282aae55 KW	139	if (adev->irq.ih.use_doorbell) {
	140	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	141	OFFSET, adev->irq.ih.doorbell_index);
	142	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	143	ENABLE, 1);
	144	} else {
	145	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	146	ENABLE, 0);
	147	}
b2b7e457	148	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
bf383fb6 AD	149	adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell,
bf383fb6 AD	150	adev->irq.ih.doorbell_index);
282aae55	151
b2b7e457	152	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
282aae55 KW	153	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
282aae55 KW	154	CLIENT18_IS_STORM_CLIENT, 1);
b2b7e457	155	WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
282aae55	156
b2b7e457	157	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
282aae55	158	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
b2b7e457	159	WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
282aae55 KW	160
	161	pci_set_master(adev->pdev);
	162
	163	/* enable interrupts */
	164	vega10_ih_enable_interrupts(adev);
	165
	166	return ret;
	167	}
	168
	169	/**
	170	* vega10_ih_irq_disable - disable interrupts
	171	*
	172	* @adev: amdgpu_device pointer
	173	*
	174	* Disable interrupts on the hw (VEGA10).
	175	*/
	176	static void vega10_ih_irq_disable(struct amdgpu_device *adev)
	177	{
	178	vega10_ih_disable_interrupts(adev);
	179
	180	/* Wait and acknowledge irq */
	181	mdelay(1);
	182	}
	183
	184	/**
	185	* vega10_ih_get_wptr - get the IH ring buffer wptr
	186	*
	187	* @adev: amdgpu_device pointer
	188	*
	189	* Get the IH ring buffer wptr from either the register
	190	* or the writeback memory buffer (VEGA10). Also check for
	191	* ring buffer overflow and deal with it.
	192	* Returns the value of the wptr.
	193	*/
	194	static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
	195	{
	196	u32 wptr, tmp;
	197
	198	if (adev->irq.ih.use_bus_addr)
	199	wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);
	200	else
	201	wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);
	202
	203	if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
	204	wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
	205
	206	/* When a ring buffer overflow happen start parsing interrupt
	207	* from the last not overwritten vector (wptr + 32). Hopefully
	208	* this should allow us to catchup.
	209	*/
	210	tmp = (wptr + 32) & adev->irq.ih.ptr_mask;
	211	dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
	212	wptr, adev->irq.ih.rptr, tmp);
	213	adev->irq.ih.rptr = tmp;
	214
7c3f2167	215	tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
282aae55	216	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
7c3f2167	217	WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
282aae55 KW	218	}
	219	return (wptr & adev->irq.ih.ptr_mask);
	220	}
	221
	222	/**
	223	* vega10_ih_decode_iv - decode an interrupt vector
	224	*
	225	* @adev: amdgpu_device pointer
	226	*
	227	* Decodes the interrupt vector at the current rptr
	228	* position and also advance the position.
	229	*/
	230	static void vega10_ih_decode_iv(struct amdgpu_device *adev,
	231	struct amdgpu_iv_entry *entry)
	232	{
	233	/* wptr/rptr are in bytes! */
	234	u32 ring_index = adev->irq.ih.rptr >> 2;
	235	uint32_t dw[8];
	236
	237	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	238	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
	239	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
	240	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	241	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	242	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
	243	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);
	244	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);
	245
	246	entry->client_id = dw[0] & 0xff;
	247	entry->src_id = (dw[0] >> 8) & 0xff;
	248	entry->ring_id = (dw[0] >> 16) & 0xff;
c4f46f22 CK	249	entry->vmid = (dw[0] >> 24) & 0xf;
c4f46f22 CK	250	entry->vmid_src = (dw[0] >> 31);
282aae55 KW	251	entry->timestamp = dw[1] \| ((u64)(dw[2] & 0xffff) << 32);
282aae55 KW	252	entry->timestamp_src = dw[2] >> 31;
3816e42f	253	entry->pasid = dw[3] & 0xffff;
282aae55 KW	254	entry->pasid_src = dw[3] >> 31;
	255	entry->src_data[0] = dw[4];
	256	entry->src_data[1] = dw[5];
	257	entry->src_data[2] = dw[6];
	258	entry->src_data[3] = dw[7];
	259
	260
	261	/* wptr/rptr are in bytes! */
	262	adev->irq.ih.rptr += 32;
	263	}
	264
	265	/**
	266	* vega10_ih_set_rptr - set the IH ring buffer rptr
	267	*
	268	* @adev: amdgpu_device pointer
	269	*
	270	* Set the IH ring buffer rptr.
	271	*/
	272	static void vega10_ih_set_rptr(struct amdgpu_device *adev)
	273	{
	274	if (adev->irq.ih.use_doorbell) {
	275	/* XXX check if swapping is necessary on BE */
	276	if (adev->irq.ih.use_bus_addr)
	277	adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
	278	else
	279	adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
	280	WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
	281	} else {
b2b7e457	282	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, adev->irq.ih.rptr);
282aae55 KW	283	}
	284	}
	285
	286	static int vega10_ih_early_init(void *handle)
	287	{
	288	struct amdgpu_device adev = (struct amdgpu_device )handle;
	289
	290	vega10_ih_set_interrupt_funcs(adev);
	291	return 0;
	292	}
	293
	294	static int vega10_ih_sw_init(void *handle)
	295	{
	296	int r;
	297	struct amdgpu_device adev = (struct amdgpu_device )handle;
	298
425c3143	299	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true);
282aae55 KW	300	if (r)
	301	return r;
	302
	303	adev->irq.ih.use_doorbell = true;
9564f192	304	adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
282aae55 KW	305
	306	r = amdgpu_irq_init(adev);
	307
	308	return r;
	309	}
	310
	311	static int vega10_ih_sw_fini(void *handle)
	312	{
	313	struct amdgpu_device adev = (struct amdgpu_device )handle;
	314
	315	amdgpu_irq_fini(adev);
425c3143	316	amdgpu_ih_ring_fini(adev, &adev->irq.ih);
282aae55 KW	317
	318	return 0;
	319	}
	320
	321	static int vega10_ih_hw_init(void *handle)
	322	{
	323	int r;
	324	struct amdgpu_device adev = (struct amdgpu_device )handle;
	325
	326	r = vega10_ih_irq_init(adev);
	327	if (r)
	328	return r;
	329
	330	return 0;
	331	}
	332
	333	static int vega10_ih_hw_fini(void *handle)
	334	{
	335	struct amdgpu_device adev = (struct amdgpu_device )handle;
	336
	337	vega10_ih_irq_disable(adev);
	338
	339	return 0;
	340	}
	341
	342	static int vega10_ih_suspend(void *handle)
	343	{
	344	struct amdgpu_device adev = (struct amdgpu_device )handle;
	345
	346	return vega10_ih_hw_fini(adev);
	347	}
	348
	349	static int vega10_ih_resume(void *handle)
	350	{
	351	struct amdgpu_device adev = (struct amdgpu_device )handle;
	352
	353	return vega10_ih_hw_init(adev);
	354	}
	355
	356	static bool vega10_ih_is_idle(void *handle)
	357	{
	358	/* todo */
	359	return true;
	360	}
	361
	362	static int vega10_ih_wait_for_idle(void *handle)
	363	{
	364	/* todo */
	365	return -ETIMEDOUT;
	366	}
	367
	368	static int vega10_ih_soft_reset(void *handle)
	369	{
	370	/* todo */
	371
	372	return 0;
	373	}
	374
	375	static int vega10_ih_set_clockgating_state(void *handle,
	376	enum amd_clockgating_state state)
	377	{
	378	return 0;
	379	}
	380
381	static int vega10_ih_set_powergating_state(void *handle,
382	enum amd_powergating_state state)
383	{
384	return 0;
385	}
386
387	const struct amd_ip_funcs vega10_ih_ip_funcs = {
388	.name = "vega10_ih",
389	.early_init = vega10_ih_early_init,
390	.late_init = NULL,
391	.sw_init = vega10_ih_sw_init,
392	.sw_fini = vega10_ih_sw_fini,
393	.hw_init = vega10_ih_hw_init,
394	.hw_fini = vega10_ih_hw_fini,
395	.suspend = vega10_ih_suspend,
396	.resume = vega10_ih_resume,
397	.is_idle = vega10_ih_is_idle,
398	.wait_for_idle = vega10_ih_wait_for_idle,
399	.soft_reset = vega10_ih_soft_reset,
400	.set_clockgating_state = vega10_ih_set_clockgating_state,
401	.set_powergating_state = vega10_ih_set_powergating_state,
402	};
403
404	static const struct amdgpu_ih_funcs vega10_ih_funcs = {
405	.get_wptr = vega10_ih_get_wptr,
406	.decode_iv = vega10_ih_decode_iv,
407	.set_rptr = vega10_ih_set_rptr
408	};
409
410	static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
411	{
f54b30d7	412	adev->irq.ih_funcs = &vega10_ih_funcs;
282aae55 KW	413	}
	414
	415	const struct amdgpu_ip_block_version vega10_ih_ip_block =
	416	{
	417	.type = AMD_IP_BLOCK_TYPE_IH,
	418	.major = 4,
	419	.minor = 0,
	420	.rev = 0,
	421	.funcs = &vega10_ih_ip_funcs,
	422	};