[linux-block.git] / drivers / accel / ivpu / ivpu_ipc.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2020-2023 Intel Corporation
 */

#include <linux/genalloc.h>
#include <linux/highmem.h>
#include <linux/pm_runtime.h>
#include <linux/wait.h>

#include "ivpu_drv.h"
#include "ivpu_gem.h"
#include "ivpu_hw.h"
#include "ivpu_hw_reg_io.h"
#include "ivpu_ipc.h"
#include "ivpu_jsm_msg.h"
#include "ivpu_pm.h"

#define IPC_MAX_RX_MSG	128

struct ivpu_ipc_tx_buf {
	struct ivpu_ipc_hdr ipc;
	struct vpu_jsm_msg jsm;
};

static void ivpu_ipc_msg_dump(struct ivpu_device *vdev, char *c,
			      struct ivpu_ipc_hdr *ipc_hdr, u32 vpu_addr)
{
	ivpu_dbg(vdev, IPC,
		 "%s: vpu:0x%x (data_addr:0x%08x, data_size:0x%x, channel:0x%x, src_node:0x%x, dst_node:0x%x, status:0x%x)",
		 c, vpu_addr, ipc_hdr->data_addr, ipc_hdr->data_size, ipc_hdr->channel,
		 ipc_hdr->src_node, ipc_hdr->dst_node, ipc_hdr->status);
}

static void ivpu_jsm_msg_dump(struct ivpu_device *vdev, char *c,
			      struct vpu_jsm_msg *jsm_msg, u32 vpu_addr)
{
	u32 *payload = (u32 *)&jsm_msg->payload;

	ivpu_dbg(vdev, JSM,
		 "%s: vpu:0x%08x (type:%s, status:0x%x, id: 0x%x, result: 0x%x, payload:0x%x 0x%x 0x%x 0x%x 0x%x)\n",
		 c, vpu_addr, ivpu_jsm_msg_type_to_str(jsm_msg->type),
		 jsm_msg->status, jsm_msg->request_id, jsm_msg->result,
		 payload[0], payload[1], payload[2], payload[3], payload[4]);
}

static void
ivpu_ipc_rx_mark_free(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
		      struct vpu_jsm_msg *jsm_msg)
{
	ipc_hdr->status = IVPU_IPC_HDR_FREE;
	if (jsm_msg)
		jsm_msg->status = VPU_JSM_MSG_FREE;
	wmb(); /* Flush WC buffers for message statuses */
}

static void ivpu_ipc_mem_fini(struct ivpu_device *vdev)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;

	ivpu_bo_free(ipc->mem_rx);
	ivpu_bo_free(ipc->mem_tx);
}

static int
ivpu_ipc_tx_prepare(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
		    struct vpu_jsm_msg *req)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;
	struct ivpu_ipc_tx_buf *tx_buf;
	u32 tx_buf_vpu_addr;
	u32 jsm_vpu_addr;

	tx_buf_vpu_addr = gen_pool_alloc(ipc->mm_tx, sizeof(*tx_buf));
	if (!tx_buf_vpu_addr) {
		ivpu_err_ratelimited(vdev, "Failed to reserve IPC buffer, size %ld\n",
				     sizeof(*tx_buf));
		return -ENOMEM;
	}

	tx_buf = ivpu_to_cpu_addr(ipc->mem_tx, tx_buf_vpu_addr);
	if (drm_WARN_ON(&vdev->drm, !tx_buf)) {
		gen_pool_free(ipc->mm_tx, tx_buf_vpu_addr, sizeof(*tx_buf));
		return -EIO;
	}

	jsm_vpu_addr = tx_buf_vpu_addr + offsetof(struct ivpu_ipc_tx_buf, jsm);

	if (tx_buf->ipc.status != IVPU_IPC_HDR_FREE)
		ivpu_warn_ratelimited(vdev, "IPC message vpu:0x%x not released by firmware\n",
				      tx_buf_vpu_addr);

	if (tx_buf->jsm.status != VPU_JSM_MSG_FREE)
		ivpu_warn_ratelimited(vdev, "JSM message vpu:0x%x not released by firmware\n",
				      jsm_vpu_addr);

	memset(tx_buf, 0, sizeof(*tx_buf));
	tx_buf->ipc.data_addr = jsm_vpu_addr;
	/* TODO: Set data_size to actual JSM message size, not union of all messages */
	tx_buf->ipc.data_size = sizeof(*req);
	tx_buf->ipc.channel = cons->channel;
	tx_buf->ipc.src_node = 0;
	tx_buf->ipc.dst_node = 1;
	tx_buf->ipc.status = IVPU_IPC_HDR_ALLOCATED;
	tx_buf->jsm.type = req->type;
	tx_buf->jsm.status = VPU_JSM_MSG_ALLOCATED;
	tx_buf->jsm.payload = req->payload;

	req->request_id = atomic_inc_return(&ipc->request_id);
	tx_buf->jsm.request_id = req->request_id;
	cons->request_id = req->request_id;
	wmb(); /* Flush WC buffers for IPC, JSM msgs */

	cons->tx_vpu_addr = tx_buf_vpu_addr;

	ivpu_jsm_msg_dump(vdev, "TX", &tx_buf->jsm, jsm_vpu_addr);
	ivpu_ipc_msg_dump(vdev, "TX", &tx_buf->ipc, tx_buf_vpu_addr);

	return 0;
}

static void ivpu_ipc_tx_release(struct ivpu_device *vdev, u32 vpu_addr)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;

	if (vpu_addr)
		gen_pool_free(ipc->mm_tx, vpu_addr, sizeof(struct ivpu_ipc_tx_buf));
}

static void ivpu_ipc_tx(struct ivpu_device *vdev, u32 vpu_addr)
{
	ivpu_hw_reg_ipc_tx_set(vdev, vpu_addr);
}

static void
ivpu_ipc_rx_msg_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
		    struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;
	struct ivpu_ipc_rx_msg *rx_msg;

	lockdep_assert_held(&ipc->cons_lock);
	lockdep_assert_irqs_disabled();

	rx_msg = kzalloc(sizeof(*rx_msg), GFP_ATOMIC);
	if (!rx_msg) {
		ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
		return;
	}

	atomic_inc(&ipc->rx_msg_count);

	rx_msg->ipc_hdr = ipc_hdr;
	rx_msg->jsm_msg = jsm_msg;
	rx_msg->callback = cons->rx_callback;

	if (rx_msg->callback) {
		list_add_tail(&rx_msg->link, &ipc->cb_msg_list);
	} else {
		spin_lock(&cons->rx_lock);
		list_add_tail(&rx_msg->link, &cons->rx_msg_list);
		spin_unlock(&cons->rx_lock);
		wake_up(&cons->rx_msg_wq);
	}
}

static void
ivpu_ipc_rx_msg_del(struct ivpu_device *vdev, struct ivpu_ipc_rx_msg *rx_msg)
{
	list_del(&rx_msg->link);
	ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
	atomic_dec(&vdev->ipc->rx_msg_count);
	kfree(rx_msg);
}

void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
			   u32 channel, ivpu_ipc_rx_callback_t rx_callback)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;

	INIT_LIST_HEAD(&cons->link);
	cons->channel = channel;
	cons->tx_vpu_addr = 0;
	cons->request_id = 0;
	cons->aborted = false;
	cons->rx_callback = rx_callback;
	spin_lock_init(&cons->rx_lock);
	INIT_LIST_HEAD(&cons->rx_msg_list);
	init_waitqueue_head(&cons->rx_msg_wq);

	spin_lock_irq(&ipc->cons_lock);
	list_add_tail(&cons->link, &ipc->cons_list);
	spin_unlock_irq(&ipc->cons_lock);
}

void ivpu_ipc_consumer_del(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;
	struct ivpu_ipc_rx_msg *rx_msg, *r;

	spin_lock_irq(&ipc->cons_lock);
	list_del(&cons->link);
	spin_unlock_irq(&ipc->cons_lock);

	spin_lock_irq(&cons->rx_lock);
	list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link)
		ivpu_ipc_rx_msg_del(vdev, rx_msg);
	spin_unlock_irq(&cons->rx_lock);

	ivpu_ipc_tx_release(vdev, cons->tx_vpu_addr);
}

static int
ivpu_ipc_send(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, struct vpu_jsm_msg *req)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;
	int ret;

	mutex_lock(&ipc->lock);

	if (!ipc->on) {
		ret = -EAGAIN;
		goto unlock;
	}

	ret = ivpu_ipc_tx_prepare(vdev, cons, req);
	if (ret)
		goto unlock;

	ivpu_ipc_tx(vdev, cons->tx_vpu_addr);

unlock:
	mutex_unlock(&ipc->lock);
	return ret;
}

static bool ivpu_ipc_rx_need_wakeup(struct ivpu_ipc_consumer *cons)
{
	bool ret;

	spin_lock_irq(&cons->rx_lock);
	ret = !list_empty(&cons->rx_msg_list) || cons->aborted;
	spin_unlock_irq(&cons->rx_lock);

	return ret;
}

int ivpu_ipc_receive(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
		     struct ivpu_ipc_hdr *ipc_buf,
		     struct vpu_jsm_msg *jsm_msg, unsigned long timeout_ms)
{
	struct ivpu_ipc_rx_msg *rx_msg;
	int wait_ret, ret = 0;

	if (drm_WARN_ONCE(&vdev->drm, cons->rx_callback, "Consumer works only in async mode\n"))
		return -EINVAL;

	wait_ret = wait_event_timeout(cons->rx_msg_wq,
				      ivpu_ipc_rx_need_wakeup(cons),
				      msecs_to_jiffies(timeout_ms));

	if (wait_ret == 0)
		return -ETIMEDOUT;

	spin_lock_irq(&cons->rx_lock);
	if (cons->aborted) {
		spin_unlock_irq(&cons->rx_lock);
		return -ECANCELED;
	}
	rx_msg = list_first_entry_or_null(&cons->rx_msg_list, struct ivpu_ipc_rx_msg, link);
	if (!rx_msg) {
		spin_unlock_irq(&cons->rx_lock);
		return -EAGAIN;
	}

	if (ipc_buf)
		memcpy(ipc_buf, rx_msg->ipc_hdr, sizeof(*ipc_buf));
	if (rx_msg->jsm_msg) {
		u32 size = min_t(int, rx_msg->ipc_hdr->data_size, sizeof(*jsm_msg));

		if (rx_msg->jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
			ivpu_dbg(vdev, IPC, "IPC resp result error: %d\n", rx_msg->jsm_msg->result);
			ret = -EBADMSG;
		}

		if (jsm_msg)
			memcpy(jsm_msg, rx_msg->jsm_msg, size);
	}

	ivpu_ipc_rx_msg_del(vdev, rx_msg);
	spin_unlock_irq(&cons->rx_lock);
	return ret;
}

static int
ivpu_ipc_send_receive_internal(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
			       enum vpu_ipc_msg_type expected_resp_type,
			       struct vpu_jsm_msg *resp, u32 channel,
			       unsigned long timeout_ms)
{
	struct ivpu_ipc_consumer cons;
	int ret;

	ivpu_ipc_consumer_add(vdev, &cons, channel, NULL);

	ret = ivpu_ipc_send(vdev, &cons, req);
	if (ret) {
		ivpu_warn_ratelimited(vdev, "IPC send failed: %d\n", ret);
		goto consumer_del;
	}

	ret = ivpu_ipc_receive(vdev, &cons, NULL, resp, timeout_ms);
	if (ret) {
		ivpu_warn_ratelimited(vdev, "IPC receive failed: type %s, ret %d\n",
				      ivpu_jsm_msg_type_to_str(req->type), ret);
		goto consumer_del;
	}

	if (resp->type != expected_resp_type) {
		ivpu_warn_ratelimited(vdev, "Invalid JSM response type: 0x%x\n", resp->type);
		ret = -EBADE;
	}

consumer_del:
	ivpu_ipc_consumer_del(vdev, &cons);
	return ret;
}

int ivpu_ipc_send_receive_active(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
				 enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
				 u32 channel, unsigned long timeout_ms)
{
	struct vpu_jsm_msg hb_req = { .type = VPU_JSM_MSG_QUERY_ENGINE_HB };
	struct vpu_jsm_msg hb_resp;
	int ret, hb_ret;

	drm_WARN_ON(&vdev->drm, pm_runtime_status_suspended(vdev->drm.dev));

	ret = ivpu_ipc_send_receive_internal(vdev, req, expected_resp, resp, channel, timeout_ms);
	if (ret != -ETIMEDOUT)
		return ret;

	hb_ret = ivpu_ipc_send_receive_internal(vdev, &hb_req, VPU_JSM_MSG_QUERY_ENGINE_HB_DONE,
						&hb_resp, VPU_IPC_CHAN_ASYNC_CMD,
						vdev->timeout.jsm);
	if (hb_ret == -ETIMEDOUT)
		ivpu_pm_trigger_recovery(vdev, "IPC timeout");

	return ret;
}

int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
			  enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
			  u32 channel, unsigned long timeout_ms)
{
	int ret;

	ret = ivpu_rpm_get(vdev);
	if (ret < 0)
		return ret;

	ret = ivpu_ipc_send_receive_active(vdev, req, expected_resp, resp, channel, timeout_ms);

	ivpu_rpm_put(vdev);
	return ret;
}

static bool
ivpu_ipc_match_consumer(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
			struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg)
{
	if (cons->channel != ipc_hdr->channel)
		return false;

	if (!jsm_msg || jsm_msg->request_id == cons->request_id)
		return true;

	return false;
}

void ivpu_ipc_irq_handler(struct ivpu_device *vdev, bool *wake_thread)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;
	struct ivpu_ipc_consumer *cons;
	struct ivpu_ipc_hdr *ipc_hdr;
	struct vpu_jsm_msg *jsm_msg;
	unsigned long flags;
	bool dispatched;
	u32 vpu_addr;

	/*
	 * Driver needs to purge all messages from IPC FIFO to clear IPC interrupt.
	 * Without purge IPC FIFO to 0 next IPC interrupts won't be generated.
	 */
	while (ivpu_hw_reg_ipc_rx_count_get(vdev)) {
		vpu_addr = ivpu_hw_reg_ipc_rx_addr_get(vdev);
		if (vpu_addr == REG_IO_ERROR) {
			ivpu_err_ratelimited(vdev, "Failed to read IPC rx addr register\n");
			return;
		}

		ipc_hdr = ivpu_to_cpu_addr(ipc->mem_rx, vpu_addr);
		if (!ipc_hdr) {
			ivpu_warn_ratelimited(vdev, "IPC msg 0x%x out of range\n", vpu_addr);
			continue;
		}
		ivpu_ipc_msg_dump(vdev, "RX", ipc_hdr, vpu_addr);

		jsm_msg = NULL;
		if (ipc_hdr->channel != IVPU_IPC_CHAN_BOOT_MSG) {
			jsm_msg = ivpu_to_cpu_addr(ipc->mem_rx, ipc_hdr->data_addr);
			if (!jsm_msg) {
				ivpu_warn_ratelimited(vdev, "JSM msg 0x%x out of range\n",
						      ipc_hdr->data_addr);
				ivpu_ipc_rx_mark_free(vdev, ipc_hdr, NULL);
				continue;
			}
			ivpu_jsm_msg_dump(vdev, "RX", jsm_msg, ipc_hdr->data_addr);
		}

		if (atomic_read(&ipc->rx_msg_count) > IPC_MAX_RX_MSG) {
			ivpu_warn_ratelimited(vdev, "IPC RX msg dropped, msg count %d\n",
					      IPC_MAX_RX_MSG);
			ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
			continue;
		}

		dispatched = false;
		spin_lock_irqsave(&ipc->cons_lock, flags);
		list_for_each_entry(cons, &ipc->cons_list, link) {
			if (ivpu_ipc_match_consumer(vdev, cons, ipc_hdr, jsm_msg)) {
				ivpu_ipc_rx_msg_add(vdev, cons, ipc_hdr, jsm_msg);
				dispatched = true;
				break;
			}
		}
		spin_unlock_irqrestore(&ipc->cons_lock, flags);

		if (!dispatched) {
			ivpu_dbg(vdev, IPC, "IPC RX msg 0x%x dropped (no consumer)\n", vpu_addr);
			ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
		}
	}

	if (wake_thread)
		*wake_thread = !list_empty(&ipc->cb_msg_list);
}

irqreturn_t ivpu_ipc_irq_thread_handler(struct ivpu_device *vdev)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;
	struct ivpu_ipc_rx_msg *rx_msg, *r;
	struct list_head cb_msg_list;

	INIT_LIST_HEAD(&cb_msg_list);

	spin_lock_irq(&ipc->cons_lock);
	list_splice_tail_init(&ipc->cb_msg_list, &cb_msg_list);
	spin_unlock_irq(&ipc->cons_lock);

	list_for_each_entry_safe(rx_msg, r, &cb_msg_list, link) {
		rx_msg->callback(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
		ivpu_ipc_rx_msg_del(vdev, rx_msg);
	}

	return IRQ_HANDLED;
}

int ivpu_ipc_init(struct ivpu_device *vdev)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;
	int ret;

	ipc->mem_tx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);
	if (!ipc->mem_tx) {
		ivpu_err(vdev, "Failed to allocate mem_tx\n");
		return -ENOMEM;
	}

	ipc->mem_rx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);
	if (!ipc->mem_rx) {
		ivpu_err(vdev, "Failed to allocate mem_rx\n");
		ret = -ENOMEM;
		goto err_free_tx;
	}

	ipc->mm_tx = devm_gen_pool_create(vdev->drm.dev, __ffs(IVPU_IPC_ALIGNMENT),
					  -1, "TX_IPC_JSM");
	if (IS_ERR(ipc->mm_tx)) {
		ret = PTR_ERR(ipc->mm_tx);
		ivpu_err(vdev, "Failed to create gen pool, %pe\n", ipc->mm_tx);
		goto err_free_rx;
	}

	ret = gen_pool_add(ipc->mm_tx, ipc->mem_tx->vpu_addr, ivpu_bo_size(ipc->mem_tx), -1);
	if (ret) {
		ivpu_err(vdev, "gen_pool_add failed, ret %d\n", ret);
		goto err_free_rx;
	}

	spin_lock_init(&ipc->cons_lock);
	INIT_LIST_HEAD(&ipc->cons_list);
	INIT_LIST_HEAD(&ipc->cb_msg_list);
	drmm_mutex_init(&vdev->drm, &ipc->lock);
	ivpu_ipc_reset(vdev);
	return 0;

err_free_rx:
	ivpu_bo_free(ipc->mem_rx);
err_free_tx:
	ivpu_bo_free(ipc->mem_tx);
	return ret;
}

void ivpu_ipc_fini(struct ivpu_device *vdev)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;

	drm_WARN_ON(&vdev->drm, ipc->on);
	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cons_list));
	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list));
	drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0);

	ivpu_ipc_mem_fini(vdev);
}

void ivpu_ipc_enable(struct ivpu_device *vdev)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;

	mutex_lock(&ipc->lock);
	ipc->on = true;
	mutex_unlock(&ipc->lock);
}

void ivpu_ipc_disable(struct ivpu_device *vdev)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;
	struct ivpu_ipc_consumer *cons, *c;
	struct ivpu_ipc_rx_msg *rx_msg, *r;

	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list));

	mutex_lock(&ipc->lock);
	ipc->on = false;
	mutex_unlock(&ipc->lock);

	spin_lock_irq(&ipc->cons_lock);
	list_for_each_entry_safe(cons, c, &ipc->cons_list, link) {
		spin_lock(&cons->rx_lock);
		if (!cons->rx_callback)
			cons->aborted = true;
		list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link)
			ivpu_ipc_rx_msg_del(vdev, rx_msg);
		spin_unlock(&cons->rx_lock);
		wake_up(&cons->rx_msg_wq);
	}
	spin_unlock_irq(&ipc->cons_lock);

	drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0);
}

void ivpu_ipc_reset(struct ivpu_device *vdev)
{
	struct ivpu_ipc_info *ipc = vdev->ipc;

	mutex_lock(&ipc->lock);
	drm_WARN_ON(&vdev->drm, ipc->on);

	memset(ivpu_bo_vaddr(ipc->mem_tx), 0, ivpu_bo_size(ipc->mem_tx));
	memset(ivpu_bo_vaddr(ipc->mem_rx), 0, ivpu_bo_size(ipc->mem_rx));
	wmb(); /* Flush WC buffers for TX and RX rings */

	mutex_unlock(&ipc->lock);
}
Commit	Line	Data
5d7422cf JL	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2020-2023 Intel Corporation
	4	*/
	5
	6	#include <linux/genalloc.h>
	7	#include <linux/highmem.h>
c015fb6d	8	#include <linux/pm_runtime.h>
5d7422cf JL	9	#include <linux/wait.h>
	10
	11	#include "ivpu_drv.h"
	12	#include "ivpu_gem.h"
	13	#include "ivpu_hw.h"
	14	#include "ivpu_hw_reg_io.h"
	15	#include "ivpu_ipc.h"
	16	#include "ivpu_jsm_msg.h"
852be13f	17	#include "ivpu_pm.h"
5d7422cf JL	18
5d7422cf JL	19	#define IPC_MAX_RX_MSG 128
5d7422cf JL	20
	21	struct ivpu_ipc_tx_buf {
	22	struct ivpu_ipc_hdr ipc;
	23	struct vpu_jsm_msg jsm;
	24	};
	25
5d7422cf JL	26	static void ivpu_ipc_msg_dump(struct ivpu_device vdev, char c,
	27	struct ivpu_ipc_hdr *ipc_hdr, u32 vpu_addr)
	28	{
	29	ivpu_dbg(vdev, IPC,
	30	"%s: vpu:0x%x (data_addr:0x%08x, data_size:0x%x, channel:0x%x, src_node:0x%x, dst_node:0x%x, status:0x%x)",
	31	c, vpu_addr, ipc_hdr->data_addr, ipc_hdr->data_size, ipc_hdr->channel,
	32	ipc_hdr->src_node, ipc_hdr->dst_node, ipc_hdr->status);
	33	}
	34
	35	static void ivpu_jsm_msg_dump(struct ivpu_device vdev, char c,
	36	struct vpu_jsm_msg *jsm_msg, u32 vpu_addr)
	37	{
	38	u32 payload = (u32 )&jsm_msg->payload;
	39
	40	ivpu_dbg(vdev, JSM,
a3cd664e KP	41	"%s: vpu:0x%08x (type:%s, status:0x%x, id: 0x%x, result: 0x%x, payload:0x%x 0x%x 0x%x 0x%x 0x%x)\n",
	42	c, vpu_addr, ivpu_jsm_msg_type_to_str(jsm_msg->type),
	43	jsm_msg->status, jsm_msg->request_id, jsm_msg->result,
5d7422cf JL	44	payload[0], payload[1], payload[2], payload[3], payload[4]);
	45	}
	46
	47	static void
	48	ivpu_ipc_rx_mark_free(struct ivpu_device vdev, struct ivpu_ipc_hdr ipc_hdr,
	49	struct vpu_jsm_msg *jsm_msg)
	50	{
	51	ipc_hdr->status = IVPU_IPC_HDR_FREE;
	52	if (jsm_msg)
	53	jsm_msg->status = VPU_JSM_MSG_FREE;
	54	wmb(); /* Flush WC buffers for message statuses */
	55	}
	56
	57	static void ivpu_ipc_mem_fini(struct ivpu_device *vdev)
	58	{
	59	struct ivpu_ipc_info *ipc = vdev->ipc;
	60
42328003 WK	61	ivpu_bo_free(ipc->mem_rx);
42328003 WK	62	ivpu_bo_free(ipc->mem_tx);
5d7422cf JL	63	}
	64
	65	static int
	66	ivpu_ipc_tx_prepare(struct ivpu_device vdev, struct ivpu_ipc_consumer cons,
	67	struct vpu_jsm_msg *req)
	68	{
	69	struct ivpu_ipc_info *ipc = vdev->ipc;
	70	struct ivpu_ipc_tx_buf *tx_buf;
	71	u32 tx_buf_vpu_addr;
	72	u32 jsm_vpu_addr;
	73
	74	tx_buf_vpu_addr = gen_pool_alloc(ipc->mm_tx, sizeof(*tx_buf));
	75	if (!tx_buf_vpu_addr) {
276e4834 KP	76	ivpu_err_ratelimited(vdev, "Failed to reserve IPC buffer, size %ld\n",
276e4834 KP	77	sizeof(*tx_buf));
5d7422cf JL	78	return -ENOMEM;
	79	}
	80
	81	tx_buf = ivpu_to_cpu_addr(ipc->mem_tx, tx_buf_vpu_addr);
	82	if (drm_WARN_ON(&vdev->drm, !tx_buf)) {
	83	gen_pool_free(ipc->mm_tx, tx_buf_vpu_addr, sizeof(*tx_buf));
	84	return -EIO;
	85	}
	86
	87	jsm_vpu_addr = tx_buf_vpu_addr + offsetof(struct ivpu_ipc_tx_buf, jsm);
	88
	89	if (tx_buf->ipc.status != IVPU_IPC_HDR_FREE)
276e4834 KP	90	ivpu_warn_ratelimited(vdev, "IPC message vpu:0x%x not released by firmware\n",
276e4834 KP	91	tx_buf_vpu_addr);
5d7422cf JL	92
5d7422cf JL	93	if (tx_buf->jsm.status != VPU_JSM_MSG_FREE)
276e4834 KP	94	ivpu_warn_ratelimited(vdev, "JSM message vpu:0x%x not released by firmware\n",
276e4834 KP	95	jsm_vpu_addr);
5d7422cf JL	96
	97	memset(tx_buf, 0, sizeof(*tx_buf));
	98	tx_buf->ipc.data_addr = jsm_vpu_addr;
	99	/* TODO: Set data_size to actual JSM message size, not union of all messages */
	100	tx_buf->ipc.data_size = sizeof(*req);
	101	tx_buf->ipc.channel = cons->channel;
	102	tx_buf->ipc.src_node = 0;
	103	tx_buf->ipc.dst_node = 1;
	104	tx_buf->ipc.status = IVPU_IPC_HDR_ALLOCATED;
	105	tx_buf->jsm.type = req->type;
	106	tx_buf->jsm.status = VPU_JSM_MSG_ALLOCATED;
	107	tx_buf->jsm.payload = req->payload;
	108
	109	req->request_id = atomic_inc_return(&ipc->request_id);
	110	tx_buf->jsm.request_id = req->request_id;
	111	cons->request_id = req->request_id;
	112	wmb(); /* Flush WC buffers for IPC, JSM msgs */
	113
	114	cons->tx_vpu_addr = tx_buf_vpu_addr;
	115
	116	ivpu_jsm_msg_dump(vdev, "TX", &tx_buf->jsm, jsm_vpu_addr);
	117	ivpu_ipc_msg_dump(vdev, "TX", &tx_buf->ipc, tx_buf_vpu_addr);
	118
	119	return 0;
	120	}
	121
	122	static void ivpu_ipc_tx_release(struct ivpu_device *vdev, u32 vpu_addr)
	123	{
	124	struct ivpu_ipc_info *ipc = vdev->ipc;
	125
	126	if (vpu_addr)
	127	gen_pool_free(ipc->mm_tx, vpu_addr, sizeof(struct ivpu_ipc_tx_buf));
	128	}
	129
	130	static void ivpu_ipc_tx(struct ivpu_device *vdev, u32 vpu_addr)
	131	{
	132	ivpu_hw_reg_ipc_tx_set(vdev, vpu_addr);
	133	}
	134
3b434a34 JL	135	static void
	136	ivpu_ipc_rx_msg_add(struct ivpu_device vdev, struct ivpu_ipc_consumer cons,
	137	struct ivpu_ipc_hdr ipc_hdr, struct vpu_jsm_msg jsm_msg)
	138	{
	139	struct ivpu_ipc_info *ipc = vdev->ipc;
	140	struct ivpu_ipc_rx_msg *rx_msg;
	141
	142	lockdep_assert_held(&ipc->cons_lock);
	143	lockdep_assert_irqs_disabled();
	144
	145	rx_msg = kzalloc(sizeof(*rx_msg), GFP_ATOMIC);
	146	if (!rx_msg) {
	147	ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
	148	return;
	149	}
	150
	151	atomic_inc(&ipc->rx_msg_count);
	152
	153	rx_msg->ipc_hdr = ipc_hdr;
	154	rx_msg->jsm_msg = jsm_msg;
	155	rx_msg->callback = cons->rx_callback;
	156
	157	if (rx_msg->callback) {
	158	list_add_tail(&rx_msg->link, &ipc->cb_msg_list);
	159	} else {
	160	spin_lock(&cons->rx_lock);
	161	list_add_tail(&rx_msg->link, &cons->rx_msg_list);
	162	spin_unlock(&cons->rx_lock);
	163	wake_up(&cons->rx_msg_wq);
	164	}
	165	}
	166
	167	static void
	168	ivpu_ipc_rx_msg_del(struct ivpu_device vdev, struct ivpu_ipc_rx_msg rx_msg)
	169	{
	170	list_del(&rx_msg->link);
	171	ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
	172	atomic_dec(&vdev->ipc->rx_msg_count);
	173	kfree(rx_msg);
	174	}
	175
	176	void ivpu_ipc_consumer_add(struct ivpu_device vdev, struct ivpu_ipc_consumer cons,
	177	u32 channel, ivpu_ipc_rx_callback_t rx_callback)
5d7422cf JL	178	{
	179	struct ivpu_ipc_info *ipc = vdev->ipc;
	180
	181	INIT_LIST_HEAD(&cons->link);
	182	cons->channel = channel;
	183	cons->tx_vpu_addr = 0;
	184	cons->request_id = 0;
ba6b035d	185	cons->aborted = false;
3b434a34	186	cons->rx_callback = rx_callback;
043a2d5d	187	spin_lock_init(&cons->rx_lock);
5d7422cf JL	188	INIT_LIST_HEAD(&cons->rx_msg_list);
	189	init_waitqueue_head(&cons->rx_msg_wq);
	190
3b434a34	191	spin_lock_irq(&ipc->cons_lock);
5d7422cf	192	list_add_tail(&cons->link, &ipc->cons_list);
3b434a34	193	spin_unlock_irq(&ipc->cons_lock);
5d7422cf JL	194	}
	195
	196	void ivpu_ipc_consumer_del(struct ivpu_device vdev, struct ivpu_ipc_consumer cons)
	197	{
	198	struct ivpu_ipc_info *ipc = vdev->ipc;
	199	struct ivpu_ipc_rx_msg rx_msg, r;
	200
3b434a34	201	spin_lock_irq(&ipc->cons_lock);
5d7422cf	202	list_del(&cons->link);
3b434a34	203	spin_unlock_irq(&ipc->cons_lock);
5d7422cf	204
043a2d5d	205	spin_lock_irq(&cons->rx_lock);
3b434a34 JL	206	list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link)
3b434a34 JL	207	ivpu_ipc_rx_msg_del(vdev, rx_msg);
043a2d5d	208	spin_unlock_irq(&cons->rx_lock);
5d7422cf JL	209
	210	ivpu_ipc_tx_release(vdev, cons->tx_vpu_addr);
	211	}
	212
	213	static int
	214	ivpu_ipc_send(struct ivpu_device vdev, struct ivpu_ipc_consumer cons, struct vpu_jsm_msg *req)
	215	{
	216	struct ivpu_ipc_info *ipc = vdev->ipc;
	217	int ret;
	218
b563e479	219	mutex_lock(&ipc->lock);
5d7422cf JL	220
	221	if (!ipc->on) {
	222	ret = -EAGAIN;
	223	goto unlock;
	224	}
	225
	226	ret = ivpu_ipc_tx_prepare(vdev, cons, req);
	227	if (ret)
	228	goto unlock;
	229
	230	ivpu_ipc_tx(vdev, cons->tx_vpu_addr);
	231
	232	unlock:
	233	mutex_unlock(&ipc->lock);
	234	return ret;
	235	}
	236
3b434a34	237	static bool ivpu_ipc_rx_need_wakeup(struct ivpu_ipc_consumer *cons)
57c7e3e4	238	{
3b434a34	239	bool ret;
57c7e3e4	240
043a2d5d	241	spin_lock_irq(&cons->rx_lock);
3b434a34	242	ret = !list_empty(&cons->rx_msg_list) \|\| cons->aborted;
043a2d5d	243	spin_unlock_irq(&cons->rx_lock);
57c7e3e4 SG	244
	245	return ret;
	246	}
	247
5d7422cf JL	248	int ivpu_ipc_receive(struct ivpu_device vdev, struct ivpu_ipc_consumer cons,
5d7422cf JL	249	struct ivpu_ipc_hdr *ipc_buf,
3b434a34	250	struct vpu_jsm_msg *jsm_msg, unsigned long timeout_ms)
5d7422cf	251	{
5d7422cf JL	252	struct ivpu_ipc_rx_msg *rx_msg;
	253	int wait_ret, ret = 0;
	254
3b434a34 JL	255	if (drm_WARN_ONCE(&vdev->drm, cons->rx_callback, "Consumer works only in async mode\n"))
	256	return -EINVAL;
	257
b0873eea	258	wait_ret = wait_event_timeout(cons->rx_msg_wq,
3bf3e21c	259	ivpu_ipc_rx_need_wakeup(cons),
b0873eea	260	msecs_to_jiffies(timeout_ms));
5d7422cf	261
5d7422cf JL	262	if (wait_ret == 0)
	263	return -ETIMEDOUT;
	264
043a2d5d	265	spin_lock_irq(&cons->rx_lock);
b3c10b71 SG	266	if (cons->aborted) {
	267	spin_unlock_irq(&cons->rx_lock);
	268	return -ECANCELED;
	269	}
5d7422cf JL	270	rx_msg = list_first_entry_or_null(&cons->rx_msg_list, struct ivpu_ipc_rx_msg, link);
5d7422cf JL	271	if (!rx_msg) {
043a2d5d	272	spin_unlock_irq(&cons->rx_lock);
5d7422cf JL	273	return -EAGAIN;
5d7422cf JL	274	}
5d7422cf JL	275
	276	if (ipc_buf)
	277	memcpy(ipc_buf, rx_msg->ipc_hdr, sizeof(*ipc_buf));
	278	if (rx_msg->jsm_msg) {
3b434a34	279	u32 size = min_t(int, rx_msg->ipc_hdr->data_size, sizeof(*jsm_msg));
5d7422cf JL	280
	281	if (rx_msg->jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
	282	ivpu_dbg(vdev, IPC, "IPC resp result error: %d\n", rx_msg->jsm_msg->result);
	283	ret = -EBADMSG;
	284	}
	285
3b434a34 JL	286	if (jsm_msg)
3b434a34 JL	287	memcpy(jsm_msg, rx_msg->jsm_msg, size);
5d7422cf JL	288	}
5d7422cf JL	289
3b434a34 JL	290	ivpu_ipc_rx_msg_del(vdev, rx_msg);
3b434a34 JL	291	spin_unlock_irq(&cons->rx_lock);
5d7422cf JL	292	return ret;
	293	}
	294
	295	static int
	296	ivpu_ipc_send_receive_internal(struct ivpu_device vdev, struct vpu_jsm_msg req,
	297	enum vpu_ipc_msg_type expected_resp_type,
	298	struct vpu_jsm_msg *resp, u32 channel,
	299	unsigned long timeout_ms)
	300	{
	301	struct ivpu_ipc_consumer cons;
	302	int ret;
	303
3b434a34	304	ivpu_ipc_consumer_add(vdev, &cons, channel, NULL);
5d7422cf JL	305
	306	ret = ivpu_ipc_send(vdev, &cons, req);
	307	if (ret) {
276e4834	308	ivpu_warn_ratelimited(vdev, "IPC send failed: %d\n", ret);
5d7422cf JL	309	goto consumer_del;
	310	}
	311
	312	ret = ivpu_ipc_receive(vdev, &cons, NULL, resp, timeout_ms);
	313	if (ret) {
a3cd664e KP	314	ivpu_warn_ratelimited(vdev, "IPC receive failed: type %s, ret %d\n",
a3cd664e KP	315	ivpu_jsm_msg_type_to_str(req->type), ret);
5d7422cf JL	316	goto consumer_del;
	317	}
	318
	319	if (resp->type != expected_resp_type) {
276e4834	320	ivpu_warn_ratelimited(vdev, "Invalid JSM response type: 0x%x\n", resp->type);
5d7422cf JL	321	ret = -EBADE;
	322	}
	323
	324	consumer_del:
	325	ivpu_ipc_consumer_del(vdev, &cons);
	326	return ret;
	327	}
	328
45e45362 KW	329	int ivpu_ipc_send_receive_active(struct ivpu_device vdev, struct vpu_jsm_msg req,
	330	enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
	331	u32 channel, unsigned long timeout_ms)
5d7422cf JL	332	{
	333	struct vpu_jsm_msg hb_req = { .type = VPU_JSM_MSG_QUERY_ENGINE_HB };
	334	struct vpu_jsm_msg hb_resp;
852be13f JL	335	int ret, hb_ret;
852be13f JL	336
c015fb6d	337	drm_WARN_ON(&vdev->drm, pm_runtime_status_suspended(vdev->drm.dev));
5d7422cf	338
45e45362	339	ret = ivpu_ipc_send_receive_internal(vdev, req, expected_resp, resp, channel, timeout_ms);
5d7422cf	340	if (ret != -ETIMEDOUT)
45e45362	341	return ret;
5d7422cf	342
852be13f JL	343	hb_ret = ivpu_ipc_send_receive_internal(vdev, &hb_req, VPU_JSM_MSG_QUERY_ENGINE_HB_DONE,
	344	&hb_resp, VPU_IPC_CHAN_ASYNC_CMD,
	345	vdev->timeout.jsm);
27d19268 JL	346	if (hb_ret == -ETIMEDOUT)
27d19268 JL	347	ivpu_pm_trigger_recovery(vdev, "IPC timeout");
5d7422cf	348
45e45362 KW	349	return ret;
	350	}
	351
	352	int ivpu_ipc_send_receive(struct ivpu_device vdev, struct vpu_jsm_msg req,
	353	enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
	354	u32 channel, unsigned long timeout_ms)
	355	{
	356	int ret;
	357
	358	ret = ivpu_rpm_get(vdev);
	359	if (ret < 0)
	360	return ret;
	361
	362	ret = ivpu_ipc_send_receive_active(vdev, req, expected_resp, resp, channel, timeout_ms);
	363
852be13f	364	ivpu_rpm_put(vdev);
5d7422cf JL	365	return ret;
	366	}
	367
	368	static bool
	369	ivpu_ipc_match_consumer(struct ivpu_device vdev, struct ivpu_ipc_consumer cons,
	370	struct ivpu_ipc_hdr ipc_hdr, struct vpu_jsm_msg jsm_msg)
	371	{
	372	if (cons->channel != ipc_hdr->channel)
	373	return false;
	374
	375	if (!jsm_msg \|\| jsm_msg->request_id == cons->request_id)
	376	return true;
	377
	378	return false;
	379	}
	380
3b434a34	381	void ivpu_ipc_irq_handler(struct ivpu_device vdev, bool wake_thread)
5d7422cf JL	382	{
	383	struct ivpu_ipc_info *ipc = vdev->ipc;
	384	struct ivpu_ipc_consumer *cons;
	385	struct ivpu_ipc_hdr *ipc_hdr;
	386	struct vpu_jsm_msg *jsm_msg;
	387	unsigned long flags;
	388	bool dispatched;
	389	u32 vpu_addr;
	390
	391	/*
	392	* Driver needs to purge all messages from IPC FIFO to clear IPC interrupt.
	393	* Without purge IPC FIFO to 0 next IPC interrupts won't be generated.
	394	*/
	395	while (ivpu_hw_reg_ipc_rx_count_get(vdev)) {
	396	vpu_addr = ivpu_hw_reg_ipc_rx_addr_get(vdev);
	397	if (vpu_addr == REG_IO_ERROR) {
276e4834	398	ivpu_err_ratelimited(vdev, "Failed to read IPC rx addr register\n");
3b434a34	399	return;
5d7422cf JL	400	}
	401
	402	ipc_hdr = ivpu_to_cpu_addr(ipc->mem_rx, vpu_addr);
	403	if (!ipc_hdr) {
276e4834	404	ivpu_warn_ratelimited(vdev, "IPC msg 0x%x out of range\n", vpu_addr);
5d7422cf JL	405	continue;
	406	}
	407	ivpu_ipc_msg_dump(vdev, "RX", ipc_hdr, vpu_addr);
	408
	409	jsm_msg = NULL;
	410	if (ipc_hdr->channel != IVPU_IPC_CHAN_BOOT_MSG) {
	411	jsm_msg = ivpu_to_cpu_addr(ipc->mem_rx, ipc_hdr->data_addr);
	412	if (!jsm_msg) {
276e4834 KP	413	ivpu_warn_ratelimited(vdev, "JSM msg 0x%x out of range\n",
276e4834 KP	414	ipc_hdr->data_addr);
5d7422cf JL	415	ivpu_ipc_rx_mark_free(vdev, ipc_hdr, NULL);
	416	continue;
	417	}
	418	ivpu_jsm_msg_dump(vdev, "RX", jsm_msg, ipc_hdr->data_addr);
	419	}
	420
	421	if (atomic_read(&ipc->rx_msg_count) > IPC_MAX_RX_MSG) {
276e4834 KP	422	ivpu_warn_ratelimited(vdev, "IPC RX msg dropped, msg count %d\n",
276e4834 KP	423	IPC_MAX_RX_MSG);
5d7422cf JL	424	ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
	425	continue;
	426	}
	427
	428	dispatched = false;
3b434a34	429	spin_lock_irqsave(&ipc->cons_lock, flags);
5d7422cf JL	430	list_for_each_entry(cons, &ipc->cons_list, link) {
5d7422cf JL	431	if (ivpu_ipc_match_consumer(vdev, cons, ipc_hdr, jsm_msg)) {
3b434a34	432	ivpu_ipc_rx_msg_add(vdev, cons, ipc_hdr, jsm_msg);
5d7422cf JL	433	dispatched = true;
	434	break;
	435	}
	436	}
3b434a34	437	spin_unlock_irqrestore(&ipc->cons_lock, flags);
5d7422cf JL	438
	439	if (!dispatched) {
	440	ivpu_dbg(vdev, IPC, "IPC RX msg 0x%x dropped (no consumer)\n", vpu_addr);
	441	ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
	442	}
	443	}
	444
3b434a34 JL	445	if (wake_thread)
	446	*wake_thread = !list_empty(&ipc->cb_msg_list);
	447	}
	448
	449	irqreturn_t ivpu_ipc_irq_thread_handler(struct ivpu_device *vdev)
	450	{
	451	struct ivpu_ipc_info *ipc = vdev->ipc;
	452	struct ivpu_ipc_rx_msg rx_msg, r;
	453	struct list_head cb_msg_list;
	454
	455	INIT_LIST_HEAD(&cb_msg_list);
	456
	457	spin_lock_irq(&ipc->cons_lock);
	458	list_splice_tail_init(&ipc->cb_msg_list, &cb_msg_list);
	459	spin_unlock_irq(&ipc->cons_lock);
	460
	461	list_for_each_entry_safe(rx_msg, r, &cb_msg_list, link) {
	462	rx_msg->callback(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
	463	ivpu_ipc_rx_msg_del(vdev, rx_msg);
	464	}
	465
	466	return IRQ_HANDLED;
5d7422cf JL	467	}
	468
	469	int ivpu_ipc_init(struct ivpu_device *vdev)
	470	{
	471	struct ivpu_ipc_info *ipc = vdev->ipc;
0a9cd792	472	int ret;
5d7422cf	473
42328003	474	ipc->mem_tx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC \| DRM_IVPU_BO_MAPPABLE);
0a9cd792 JL	475	if (!ipc->mem_tx) {
	476	ivpu_err(vdev, "Failed to allocate mem_tx\n");
	477	return -ENOMEM;
	478	}
5d7422cf	479
42328003	480	ipc->mem_rx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC \| DRM_IVPU_BO_MAPPABLE);
0a9cd792 JL	481	if (!ipc->mem_rx) {
	482	ivpu_err(vdev, "Failed to allocate mem_rx\n");
	483	ret = -ENOMEM;
5d7422cf	484	goto err_free_tx;
0a9cd792	485	}
5d7422cf JL	486
	487	ipc->mm_tx = devm_gen_pool_create(vdev->drm.dev, __ffs(IVPU_IPC_ALIGNMENT),
	488	-1, "TX_IPC_JSM");
	489	if (IS_ERR(ipc->mm_tx)) {
	490	ret = PTR_ERR(ipc->mm_tx);
	491	ivpu_err(vdev, "Failed to create gen pool, %pe\n", ipc->mm_tx);
	492	goto err_free_rx;
	493	}
	494
a18f1724	495	ret = gen_pool_add(ipc->mm_tx, ipc->mem_tx->vpu_addr, ivpu_bo_size(ipc->mem_tx), -1);
5d7422cf JL	496	if (ret) {
	497	ivpu_err(vdev, "gen_pool_add failed, ret %d\n", ret);
	498	goto err_free_rx;
	499	}
	500
3b434a34	501	spin_lock_init(&ipc->cons_lock);
5d7422cf	502	INIT_LIST_HEAD(&ipc->cons_list);
3b434a34	503	INIT_LIST_HEAD(&ipc->cb_msg_list);
5d7422cf	504	drmm_mutex_init(&vdev->drm, &ipc->lock);
5d7422cf JL	505	ivpu_ipc_reset(vdev);
	506	return 0;
	507
	508	err_free_rx:
42328003	509	ivpu_bo_free(ipc->mem_rx);
5d7422cf	510	err_free_tx:
42328003	511	ivpu_bo_free(ipc->mem_tx);
5d7422cf JL	512	return ret;
	513	}
	514
	515	void ivpu_ipc_fini(struct ivpu_device *vdev)
	516	{
3b434a34 JL	517	struct ivpu_ipc_info *ipc = vdev->ipc;
	518
	519	drm_WARN_ON(&vdev->drm, ipc->on);
	520	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cons_list));
	521	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list));
	522	drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0);
	523
5d7422cf JL	524	ivpu_ipc_mem_fini(vdev);
	525	}
	526
	527	void ivpu_ipc_enable(struct ivpu_device *vdev)
	528	{
	529	struct ivpu_ipc_info *ipc = vdev->ipc;
	530
	531	mutex_lock(&ipc->lock);
	532	ipc->on = true;
	533	mutex_unlock(&ipc->lock);
	534	}
	535
	536	void ivpu_ipc_disable(struct ivpu_device *vdev)
	537	{
	538	struct ivpu_ipc_info *ipc = vdev->ipc;
	539	struct ivpu_ipc_consumer cons, c;
3b434a34 JL	540	struct ivpu_ipc_rx_msg rx_msg, r;
	541
	542	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list));
5d7422cf JL	543
	544	mutex_lock(&ipc->lock);
	545	ipc->on = false;
	546	mutex_unlock(&ipc->lock);
	547
3b434a34	548	spin_lock_irq(&ipc->cons_lock);
ba6b035d	549	list_for_each_entry_safe(cons, c, &ipc->cons_list, link) {
3b434a34 JL	550	spin_lock(&cons->rx_lock);
3b434a34 JL	551	if (!cons->rx_callback)
b3c10b71	552	cons->aborted = true;
3b434a34 JL	553	list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link)
	554	ivpu_ipc_rx_msg_del(vdev, rx_msg);
	555	spin_unlock(&cons->rx_lock);
5d7422cf	556	wake_up(&cons->rx_msg_wq);
ba6b035d	557	}
3b434a34 JL	558	spin_unlock_irq(&ipc->cons_lock);
	559
	560	drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0);
5d7422cf JL	561	}
	562
	563	void ivpu_ipc_reset(struct ivpu_device *vdev)
	564	{
	565	struct ivpu_ipc_info *ipc = vdev->ipc;
	566
	567	mutex_lock(&ipc->lock);
ba6b035d	568	drm_WARN_ON(&vdev->drm, ipc->on);
5d7422cf	569
a18f1724 JL	570	memset(ivpu_bo_vaddr(ipc->mem_tx), 0, ivpu_bo_size(ipc->mem_tx));
a18f1724 JL	571	memset(ivpu_bo_vaddr(ipc->mem_rx), 0, ivpu_bo_size(ipc->mem_rx));
5d7422cf JL	572	wmb(); /* Flush WC buffers for TX and RX rings */
	573
	574	mutex_unlock(&ipc->lock);
	575	}