[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / vega10_ih.c

/*
 * Copyright 2016 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */
#include <drm/drmP.h>
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "soc15.h"


#include "vega10/soc15ip.h"
#include "vega10/OSSSYS/osssys_4_0_offset.h"
#include "vega10/OSSSYS/osssys_4_0_sh_mask.h"

#include "soc15_common.h"
#include "vega10_ih.h"


static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);

/**
 * vega10_ih_enable_interrupts - Enable the interrupt ring buffer
 *
 * @adev: amdgpu_device pointer
 *
 * Enable the interrupt ring buffer (VEGA10).
 */
static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
{
	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));

	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
	adev->irq.ih.enabled = true;
}

/**
 * vega10_ih_disable_interrupts - Disable the interrupt ring buffer
 *
 * @adev: amdgpu_device pointer
 *
 * Disable the interrupt ring buffer (VEGA10).
 */
static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
{
	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));

	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
	/* set rptr, wptr to 0 */
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);
	adev->irq.ih.enabled = false;
	adev->irq.ih.rptr = 0;
}

/**
 * vega10_ih_irq_init - init and enable the interrupt ring
 *
 * @adev: amdgpu_device pointer
 *
 * Allocate a ring buffer for the interrupt controller,
 * enable the RLC, disable interrupts, enable the IH
 * ring buffer and enable it (VI).
 * Called at device load and reume.
 * Returns 0 for success, errors for failure.
 */
static int vega10_ih_irq_init(struct amdgpu_device *adev)
{
	int ret = 0;
	int rb_bufsz;
	u32 ih_rb_cntl, ih_doorbell_rtpr;
	u32 tmp;
	u64 wptr_off;

	/* disable irqs */
	vega10_ih_disable_interrupts(adev);

	if (adev->flags & AMD_IS_APU)
		nbio_v7_0_ih_control(adev);
	else
		nbio_v6_1_ih_control(adev);

	ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
	if (adev->irq.ih.use_bus_addr) {
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.rb_dma_addr >> 8);
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
	} else {
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.gpu_addr >> 8);
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), (adev->irq.ih.gpu_addr >> 40) & 0xff);
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
	}
	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);

	if (adev->irq.msi_enabled)
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);

	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);

	/* set the writeback address whether it's enabled or not */
	if (adev->irq.ih.use_bus_addr)
		wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
	else
		wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO), lower_32_bits(wptr_off));
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI), upper_32_bits(wptr_off) & 0xFF);

	/* set rptr, wptr to 0 */
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);

	ih_doorbell_rtpr = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR));
	if (adev->irq.ih.use_doorbell) {
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 OFFSET, adev->irq.ih.doorbell_index);
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 ENABLE, 1);
	} else {
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 ENABLE, 0);
	}
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR), ih_doorbell_rtpr);
	if (adev->flags & AMD_IS_APU)
		nbio_v7_0_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
	else
		nbio_v6_1_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);

	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL));
	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
			    CLIENT18_IS_STORM_CLIENT, 1);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL), tmp);

	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL));
	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL), tmp);

	pci_set_master(adev->pdev);

	/* enable interrupts */
	vega10_ih_enable_interrupts(adev);

	return ret;
}

/**
 * vega10_ih_irq_disable - disable interrupts
 *
 * @adev: amdgpu_device pointer
 *
 * Disable interrupts on the hw (VEGA10).
 */
static void vega10_ih_irq_disable(struct amdgpu_device *adev)
{
	vega10_ih_disable_interrupts(adev);

	/* Wait and acknowledge irq */
	mdelay(1);
}

/**
 * vega10_ih_get_wptr - get the IH ring buffer wptr
 *
 * @adev: amdgpu_device pointer
 *
 * Get the IH ring buffer wptr from either the register
 * or the writeback memory buffer (VEGA10).  Also check for
 * ring buffer overflow and deal with it.
 * Returns the value of the wptr.
 */
static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
{
	u32 wptr, tmp;

	if (adev->irq.ih.use_bus_addr)
		wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);
	else
		wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);

	if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
		wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);

		/* When a ring buffer overflow happen start parsing interrupt
		 * from the last not overwritten vector (wptr + 32). Hopefully
		 * this should allow us to catchup.
		 */
		tmp = (wptr + 32) & adev->irq.ih.ptr_mask;
		dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
			wptr, adev->irq.ih.rptr, tmp);
		adev->irq.ih.rptr = tmp;

		tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
	}
	return (wptr & adev->irq.ih.ptr_mask);
}

/**
 * vega10_ih_prescreen_iv - prescreen an interrupt vector
 *
 * @adev: amdgpu_device pointer
 *
 * Returns true if the interrupt vector should be further processed.
 */
static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
{
	u32 ring_index = adev->irq.ih.rptr >> 2;
	u32 dw0, dw3, dw4, dw5;
	u16 pasid;
	u64 addr, key;
	struct amdgpu_vm *vm;
	int r;

	dw0 = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	dw3 = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	dw4 = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	dw5 = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);

	/* Filter retry page faults, let only the first one pass. If
	 * there are too many outstanding faults, ignore them until
	 * some faults get cleared.
	 */
	switch (dw0 & 0xff) {
	case AMDGPU_IH_CLIENTID_VMC:
	case AMDGPU_IH_CLIENTID_UTCL2:
		break;
	default:
		/* Not a VM fault */
		return true;
	}

	/* Not a retry fault */
	if (!(dw5 & 0x80))
		return true;

	pasid = dw3 & 0xffff;
	/* No PASID, can't identify faulting process */
	if (!pasid)
		return true;

	addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12);
	key = AMDGPU_VM_FAULT(pasid, addr);
	r = amdgpu_ih_add_fault(adev, key);

	/* Hash table is full or the fault is already being processed,
	 * ignore further page faults
	 */
	if (r != 0)
		goto ignore_iv;

	/* Track retry faults in per-VM fault FIFO. */
	spin_lock(&adev->vm_manager.pasid_lock);
	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
	spin_unlock(&adev->vm_manager.pasid_lock);
	if (WARN_ON_ONCE(!vm)) {
		/* VM not found, process it normally */
		amdgpu_ih_clear_fault(adev, key);
		return true;
	}
	/* No locking required with single writer and single reader */
	r = kfifo_put(&vm->faults, key);
	if (!r) {
		/* FIFO is full. Ignore it until there is space */
		amdgpu_ih_clear_fault(adev, key);
		goto ignore_iv;
	}

	/* It's the first fault for this address, process it normally */
	return true;

ignore_iv:
	adev->irq.ih.rptr += 32;
	return false;
}

/**
 * vega10_ih_decode_iv - decode an interrupt vector
 *
 * @adev: amdgpu_device pointer
 *
 * Decodes the interrupt vector at the current rptr
 * position and also advance the position.
 */
static void vega10_ih_decode_iv(struct amdgpu_device *adev,
				 struct amdgpu_iv_entry *entry)
{
	/* wptr/rptr are in bytes! */
	u32 ring_index = adev->irq.ih.rptr >> 2;
	uint32_t dw[8];

	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);
	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);

	entry->client_id = dw[0] & 0xff;
	entry->src_id = (dw[0] >> 8) & 0xff;
	entry->ring_id = (dw[0] >> 16) & 0xff;
	entry->vm_id = (dw[0] >> 24) & 0xf;
	entry->vm_id_src = (dw[0] >> 31);
	entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
	entry->timestamp_src = dw[2] >> 31;
	entry->pas_id = dw[3] & 0xffff;
	entry->pasid_src = dw[3] >> 31;
	entry->src_data[0] = dw[4];
	entry->src_data[1] = dw[5];
	entry->src_data[2] = dw[6];
	entry->src_data[3] = dw[7];


	/* wptr/rptr are in bytes! */
	adev->irq.ih.rptr += 32;
}

/**
 * vega10_ih_set_rptr - set the IH ring buffer rptr
 *
 * @adev: amdgpu_device pointer
 *
 * Set the IH ring buffer rptr.
 */
static void vega10_ih_set_rptr(struct amdgpu_device *adev)
{
	if (adev->irq.ih.use_doorbell) {
		/* XXX check if swapping is necessary on BE */
		if (adev->irq.ih.use_bus_addr)
			adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
		else
			adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
		WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
	} else {
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), adev->irq.ih.rptr);
	}
}

static int vega10_ih_early_init(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	vega10_ih_set_interrupt_funcs(adev);
	return 0;
}

static int vega10_ih_sw_init(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = amdgpu_ih_ring_init(adev, 256 * 1024, true);
	if (r)
		return r;

	adev->irq.ih.use_doorbell = true;
	adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;

	adev->irq.ih.faults = kmalloc(sizeof(*adev->irq.ih.faults), GFP_KERNEL);
	if (!adev->irq.ih.faults)
		return -ENOMEM;
	INIT_CHASH_TABLE(adev->irq.ih.faults->hash,
			 AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
	spin_lock_init(&adev->irq.ih.faults->lock);
	adev->irq.ih.faults->count = 0;

	r = amdgpu_irq_init(adev);

	return r;
}

static int vega10_ih_sw_fini(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	amdgpu_irq_fini(adev);
	amdgpu_ih_ring_fini(adev);

	kfree(adev->irq.ih.faults);
	adev->irq.ih.faults = NULL;

	return 0;
}

static int vega10_ih_hw_init(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = vega10_ih_irq_init(adev);
	if (r)
		return r;

	return 0;
}

static int vega10_ih_hw_fini(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	vega10_ih_irq_disable(adev);

	return 0;
}

static int vega10_ih_suspend(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	return vega10_ih_hw_fini(adev);
}

static int vega10_ih_resume(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	return vega10_ih_hw_init(adev);
}

static bool vega10_ih_is_idle(void *handle)
{
	/* todo */
	return true;
}

static int vega10_ih_wait_for_idle(void *handle)
{
	/* todo */
	return -ETIMEDOUT;
}

static int vega10_ih_soft_reset(void *handle)
{
	/* todo */

	return 0;
}

static int vega10_ih_set_clockgating_state(void *handle,
					  enum amd_clockgating_state state)
{
	return 0;
}

static int vega10_ih_set_powergating_state(void *handle,
					  enum amd_powergating_state state)
{
	return 0;
}

const struct amd_ip_funcs vega10_ih_ip_funcs = {
	.name = "vega10_ih",
	.early_init = vega10_ih_early_init,
	.late_init = NULL,
	.sw_init = vega10_ih_sw_init,
	.sw_fini = vega10_ih_sw_fini,
	.hw_init = vega10_ih_hw_init,
	.hw_fini = vega10_ih_hw_fini,
	.suspend = vega10_ih_suspend,
	.resume = vega10_ih_resume,
	.is_idle = vega10_ih_is_idle,
	.wait_for_idle = vega10_ih_wait_for_idle,
	.soft_reset = vega10_ih_soft_reset,
	.set_clockgating_state = vega10_ih_set_clockgating_state,
	.set_powergating_state = vega10_ih_set_powergating_state,
};

static const struct amdgpu_ih_funcs vega10_ih_funcs = {
	.get_wptr = vega10_ih_get_wptr,
	.prescreen_iv = vega10_ih_prescreen_iv,
	.decode_iv = vega10_ih_decode_iv,
	.set_rptr = vega10_ih_set_rptr
};

static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
{
	if (adev->irq.ih_funcs == NULL)
		adev->irq.ih_funcs = &vega10_ih_funcs;
}

const struct amdgpu_ip_block_version vega10_ih_ip_block =
{
	.type = AMD_IP_BLOCK_TYPE_IH,
	.major = 4,
	.minor = 0,
	.rev = 0,
	.funcs = &vega10_ih_ip_funcs,
};
Commit	Line	Data
282aae55 KW	1	/*
	2	* Copyright 2016 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*
	22	*/
248a1d6f	23	#include <drm/drmP.h>
282aae55 KW	24	#include "amdgpu.h"
	25	#include "amdgpu_ih.h"
	26	#include "soc15.h"
	27
	28
	29	#include "vega10/soc15ip.h"
	30	#include "vega10/OSSSYS/osssys_4_0_offset.h"
	31	#include "vega10/OSSSYS/osssys_4_0_sh_mask.h"
	32
	33	#include "soc15_common.h"
	34	#include "vega10_ih.h"
	35
	36
	37
	38	static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
	39
	40	/**
	41	* vega10_ih_enable_interrupts - Enable the interrupt ring buffer
	42	*
	43	* @adev: amdgpu_device pointer
	44	*
	45	* Enable the interrupt ring buffer (VEGA10).
	46	*/
	47	static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
	48	{
	49	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
	50
	51	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
	52	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
	53	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
	54	adev->irq.ih.enabled = true;
	55	}
	56
	57	/**
	58	* vega10_ih_disable_interrupts - Disable the interrupt ring buffer
	59	*
	60	* @adev: amdgpu_device pointer
	61	*
	62	* Disable the interrupt ring buffer (VEGA10).
	63	*/
	64	static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
	65	{
	66	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
	67
	68	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
	69	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
	70	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
	71	/* set rptr, wptr to 0 */
	72	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
	73	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);
	74	adev->irq.ih.enabled = false;
	75	adev->irq.ih.rptr = 0;
	76	}
	77
	78	/**
	79	* vega10_ih_irq_init - init and enable the interrupt ring
	80	*
	81	* @adev: amdgpu_device pointer
	82	*
	83	* Allocate a ring buffer for the interrupt controller,
	84	* enable the RLC, disable interrupts, enable the IH
	85	* ring buffer and enable it (VI).
	86	* Called at device load and reume.
	87	* Returns 0 for success, errors for failure.
88	*/
89	static int vega10_ih_irq_init(struct amdgpu_device *adev)
90	{
91	int ret = 0;
92	int rb_bufsz;
93	u32 ih_rb_cntl, ih_doorbell_rtpr;
94	u32 tmp;
95	u64 wptr_off;
96
97	/* disable irqs */
98	vega10_ih_disable_interrupts(adev);
99
aecbe64f CZ	100	if (adev->flags & AMD_IS_APU)
	101	nbio_v7_0_ih_control(adev);
	102	else
	103	nbio_v6_1_ih_control(adev);
282aae55 KW	104
	105	ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
	106	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
	107	if (adev->irq.ih.use_bus_addr) {
	108	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.rb_dma_addr >> 8);
60508d3d	109	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
282aae55 KW	110	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
	111	} else {
	112	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.gpu_addr >> 8);
	113	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), (adev->irq.ih.gpu_addr >> 40) & 0xff);
	114	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
	115	}
	116	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
	117	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
	118	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
	119	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
	120	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
	121	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
	122	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
	123	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
	124	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
	125
	126	if (adev->irq.msi_enabled)
	127	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);
	128
	129	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
	130
	131	/* set the writeback address whether it's enabled or not */
	132	if (adev->irq.ih.use_bus_addr)
	133	wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
	134	else
	135	wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
	136	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO), lower_32_bits(wptr_off));
	137	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI), upper_32_bits(wptr_off) & 0xFF);
	138
	139	/* set rptr, wptr to 0 */
	140	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
	141	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);
	142
	143	ih_doorbell_rtpr = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR));
	144	if (adev->irq.ih.use_doorbell) {
	145	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	146	OFFSET, adev->irq.ih.doorbell_index);
	147	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	148	ENABLE, 1);
	149	} else {
	150	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	151	ENABLE, 0);
	152	}
	153	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR), ih_doorbell_rtpr);
aecbe64f CZ	154	if (adev->flags & AMD_IS_APU)
	155	nbio_v7_0_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
	156	else
	157	nbio_v6_1_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
282aae55 KW	158
	159	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL));
	160	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
	161	CLIENT18_IS_STORM_CLIENT, 1);
	162	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL), tmp);
	163
	164	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL));
	165	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
	166	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL), tmp);
	167
	168	pci_set_master(adev->pdev);
	169
	170	/* enable interrupts */
	171	vega10_ih_enable_interrupts(adev);
	172
	173	return ret;
	174	}
	175
	176	/**
	177	* vega10_ih_irq_disable - disable interrupts
	178	*
	179	* @adev: amdgpu_device pointer
	180	*
	181	* Disable interrupts on the hw (VEGA10).
	182	*/
	183	static void vega10_ih_irq_disable(struct amdgpu_device *adev)
	184	{
	185	vega10_ih_disable_interrupts(adev);
	186
	187	/* Wait and acknowledge irq */
	188	mdelay(1);
	189	}
	190
	191	/**
	192	* vega10_ih_get_wptr - get the IH ring buffer wptr
	193	*
	194	* @adev: amdgpu_device pointer
	195	*
	196	* Get the IH ring buffer wptr from either the register
	197	* or the writeback memory buffer (VEGA10). Also check for
	198	* ring buffer overflow and deal with it.
	199	* Returns the value of the wptr.
	200	*/
	201	static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
	202	{
	203	u32 wptr, tmp;
	204
	205	if (adev->irq.ih.use_bus_addr)
	206	wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);
	207	else
	208	wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);
	209
	210	if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
	211	wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
	212
	213	/* When a ring buffer overflow happen start parsing interrupt
	214	* from the last not overwritten vector (wptr + 32). Hopefully
	215	* this should allow us to catchup.
	216	*/
	217	tmp = (wptr + 32) & adev->irq.ih.ptr_mask;
	218	dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
	219	wptr, adev->irq.ih.rptr, tmp);
	220	adev->irq.ih.rptr = tmp;
	221
222	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
223	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
224	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
225	}
226	return (wptr & adev->irq.ih.ptr_mask);
227	}
228
00ecd8a2 FK	229	/**
	230	* vega10_ih_prescreen_iv - prescreen an interrupt vector
	231	*
	232	* @adev: amdgpu_device pointer
	233	*
	234	* Returns true if the interrupt vector should be further processed.
	235	*/
	236	static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
	237	{
a2f14820 FK	238	u32 ring_index = adev->irq.ih.rptr >> 2;
	239	u32 dw0, dw3, dw4, dw5;
	240	u16 pasid;
	241	u64 addr, key;
	242	struct amdgpu_vm *vm;
	243	int r;
	244
	245	dw0 = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	246	dw3 = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	247	dw4 = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	248	dw5 = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
	249
	250	/* Filter retry page faults, let only the first one pass. If
	251	* there are too many outstanding faults, ignore them until
	252	* some faults get cleared.
	253	*/
	254	switch (dw0 & 0xff) {
	255	case AMDGPU_IH_CLIENTID_VMC:
	256	case AMDGPU_IH_CLIENTID_UTCL2:
	257	break;
	258	default:
	259	/* Not a VM fault */
	260	return true;
	261	}
	262
	263	/* Not a retry fault */
	264	if (!(dw5 & 0x80))
	265	return true;
	266
	267	pasid = dw3 & 0xffff;
	268	/* No PASID, can't identify faulting process */
	269	if (!pasid)
	270	return true;
	271
	272	addr = ((u64)(dw5 & 0xf) << 44) \| ((u64)dw4 << 12);
	273	key = AMDGPU_VM_FAULT(pasid, addr);
	274	r = amdgpu_ih_add_fault(adev, key);
	275
	276	/* Hash table is full or the fault is already being processed,
	277	* ignore further page faults
	278	*/
	279	if (r != 0)
	280	goto ignore_iv;
	281
	282	/* Track retry faults in per-VM fault FIFO. */
	283	spin_lock(&adev->vm_manager.pasid_lock);
	284	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
	285	spin_unlock(&adev->vm_manager.pasid_lock);
	286	if (WARN_ON_ONCE(!vm)) {
	287	/* VM not found, process it normally */
	288	amdgpu_ih_clear_fault(adev, key);
	289	return true;
	290	}
	291	/* No locking required with single writer and single reader */
	292	r = kfifo_put(&vm->faults, key);
	293	if (!r) {
	294	/* FIFO is full. Ignore it until there is space */
	295	amdgpu_ih_clear_fault(adev, key);
	296	goto ignore_iv;
	297	}
	298
	299	/* It's the first fault for this address, process it normally */
00ecd8a2	300	return true;
a2f14820 FK	301
	302	ignore_iv:
	303	adev->irq.ih.rptr += 32;
	304	return false;
00ecd8a2 FK	305	}
00ecd8a2 FK	306
282aae55 KW	307	/**
	308	* vega10_ih_decode_iv - decode an interrupt vector
	309	*
	310	* @adev: amdgpu_device pointer
	311	*
	312	* Decodes the interrupt vector at the current rptr
	313	* position and also advance the position.
	314	*/
	315	static void vega10_ih_decode_iv(struct amdgpu_device *adev,
	316	struct amdgpu_iv_entry *entry)
	317	{
	318	/* wptr/rptr are in bytes! */
	319	u32 ring_index = adev->irq.ih.rptr >> 2;
	320	uint32_t dw[8];
	321
	322	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	323	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
	324	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
	325	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	326	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	327	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
	328	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);
	329	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);
	330
	331	entry->client_id = dw[0] & 0xff;
	332	entry->src_id = (dw[0] >> 8) & 0xff;
	333	entry->ring_id = (dw[0] >> 16) & 0xff;
	334	entry->vm_id = (dw[0] >> 24) & 0xf;
	335	entry->vm_id_src = (dw[0] >> 31);
	336	entry->timestamp = dw[1] \| ((u64)(dw[2] & 0xffff) << 32);
	337	entry->timestamp_src = dw[2] >> 31;
	338	entry->pas_id = dw[3] & 0xffff;
	339	entry->pasid_src = dw[3] >> 31;
	340	entry->src_data[0] = dw[4];
	341	entry->src_data[1] = dw[5];
	342	entry->src_data[2] = dw[6];
	343	entry->src_data[3] = dw[7];
	344
	345
	346	/* wptr/rptr are in bytes! */
	347	adev->irq.ih.rptr += 32;
	348	}
	349
	350	/**
	351	* vega10_ih_set_rptr - set the IH ring buffer rptr
	352	*
	353	* @adev: amdgpu_device pointer
	354	*
	355	* Set the IH ring buffer rptr.
	356	*/
	357	static void vega10_ih_set_rptr(struct amdgpu_device *adev)
	358	{
	359	if (adev->irq.ih.use_doorbell) {
	360	/* XXX check if swapping is necessary on BE */
	361	if (adev->irq.ih.use_bus_addr)
	362	adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
	363	else
	364	adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
	365	WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
	366	} else {
	367	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), adev->irq.ih.rptr);
	368	}
	369	}
	370
371	static int vega10_ih_early_init(void *handle)
372	{
373	struct amdgpu_device adev = (struct amdgpu_device )handle;
374
375	vega10_ih_set_interrupt_funcs(adev);
376	return 0;
377	}
378
379	static int vega10_ih_sw_init(void *handle)
380	{
381	int r;
382	struct amdgpu_device adev = (struct amdgpu_device )handle;
383
384	r = amdgpu_ih_ring_init(adev, 256 * 1024, true);
385	if (r)
386	return r;
387
388	adev->irq.ih.use_doorbell = true;
389	adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;
390
a2f14820 FK	391	adev->irq.ih.faults = kmalloc(sizeof(*adev->irq.ih.faults), GFP_KERNEL);
	392	if (!adev->irq.ih.faults)
	393	return -ENOMEM;
	394	INIT_CHASH_TABLE(adev->irq.ih.faults->hash,
	395	AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
	396	spin_lock_init(&adev->irq.ih.faults->lock);
	397	adev->irq.ih.faults->count = 0;
	398
282aae55 KW	399	r = amdgpu_irq_init(adev);
	400
	401	return r;
	402	}
	403
	404	static int vega10_ih_sw_fini(void *handle)
	405	{
	406	struct amdgpu_device adev = (struct amdgpu_device )handle;
	407
	408	amdgpu_irq_fini(adev);
	409	amdgpu_ih_ring_fini(adev);
	410
a2f14820 FK	411	kfree(adev->irq.ih.faults);
	412	adev->irq.ih.faults = NULL;
	413
282aae55 KW	414	return 0;
	415	}
	416
	417	static int vega10_ih_hw_init(void *handle)
	418	{
	419	int r;
	420	struct amdgpu_device adev = (struct amdgpu_device )handle;
	421
	422	r = vega10_ih_irq_init(adev);
	423	if (r)
	424	return r;
	425
	426	return 0;
	427	}
	428
	429	static int vega10_ih_hw_fini(void *handle)
	430	{
	431	struct amdgpu_device adev = (struct amdgpu_device )handle;
	432
	433	vega10_ih_irq_disable(adev);
	434
	435	return 0;
	436	}
	437
	438	static int vega10_ih_suspend(void *handle)
	439	{
	440	struct amdgpu_device adev = (struct amdgpu_device )handle;
	441
	442	return vega10_ih_hw_fini(adev);
	443	}
	444
	445	static int vega10_ih_resume(void *handle)
	446	{
	447	struct amdgpu_device adev = (struct amdgpu_device )handle;
	448
	449	return vega10_ih_hw_init(adev);
	450	}
	451
	452	static bool vega10_ih_is_idle(void *handle)
	453	{
	454	/* todo */
	455	return true;
	456	}
	457
	458	static int vega10_ih_wait_for_idle(void *handle)
	459	{
	460	/* todo */
	461	return -ETIMEDOUT;
	462	}
	463
	464	static int vega10_ih_soft_reset(void *handle)
	465	{
	466	/* todo */
	467
	468	return 0;
	469	}
	470
	471	static int vega10_ih_set_clockgating_state(void *handle,
	472	enum amd_clockgating_state state)
	473	{
	474	return 0;
	475	}
	476
	477	static int vega10_ih_set_powergating_state(void *handle,
478	enum amd_powergating_state state)
479	{
480	return 0;
481	}
482
483	const struct amd_ip_funcs vega10_ih_ip_funcs = {
484	.name = "vega10_ih",
485	.early_init = vega10_ih_early_init,
486	.late_init = NULL,
487	.sw_init = vega10_ih_sw_init,
488	.sw_fini = vega10_ih_sw_fini,
489	.hw_init = vega10_ih_hw_init,
490	.hw_fini = vega10_ih_hw_fini,
491	.suspend = vega10_ih_suspend,
492	.resume = vega10_ih_resume,
493	.is_idle = vega10_ih_is_idle,
494	.wait_for_idle = vega10_ih_wait_for_idle,
495	.soft_reset = vega10_ih_soft_reset,
496	.set_clockgating_state = vega10_ih_set_clockgating_state,
497	.set_powergating_state = vega10_ih_set_powergating_state,
498	};
499
500	static const struct amdgpu_ih_funcs vega10_ih_funcs = {
501	.get_wptr = vega10_ih_get_wptr,
00ecd8a2	502	.prescreen_iv = vega10_ih_prescreen_iv,
282aae55 KW	503	.decode_iv = vega10_ih_decode_iv,
	504	.set_rptr = vega10_ih_set_rptr
	505	};
	506
	507	static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
	508	{
	509	if (adev->irq.ih_funcs == NULL)
	510	adev->irq.ih_funcs = &vega10_ih_funcs;
	511	}
	512
	513	const struct amdgpu_ip_block_version vega10_ih_ip_block =
	514	{
	515	.type = AMD_IP_BLOCK_TYPE_IH,
	516	.major = 4,
	517	.minor = 0,
	518	.rev = 0,
	519	.funcs = &vega10_ih_ip_funcs,
	520	};