[linux-2.6-block.git] / drivers / soc / qcom / rpmh.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
 */

#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/interrupt.h>
#include <linux/jiffies.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include <linux/wait.h>

#include <soc/qcom/rpmh.h>

#include "rpmh-internal.h"

#define RPMH_TIMEOUT_MS			msecs_to_jiffies(10000)

#define DEFINE_RPMH_MSG_ONSTACK(device, s, q, name)	\
	struct rpmh_request name = {			\
		.msg = {				\
			.state = s,			\
			.cmds = name.cmd,		\
			.num_cmds = 0,			\
			.wait_for_compl = true,		\
		},					\
		.cmd = { { 0 } },			\
		.completion = q,			\
		.dev = device,				\
		.needs_free = false,				\
	}

#define ctrlr_to_drv(ctrlr) container_of(ctrlr, struct rsc_drv, client)

/**
 * struct cache_req: the request object for caching
 *
 * @addr: the address of the resource
 * @sleep_val: the sleep vote
 * @wake_val: the wake vote
 * @list: linked list obj
 */
struct cache_req {
	u32 addr;
	u32 sleep_val;
	u32 wake_val;
	struct list_head list;
};

/**
 * struct batch_cache_req - An entry in our batch catch
 *
 * @list: linked list obj
 * @count: number of messages
 * @rpm_msgs: the messages
 */

struct batch_cache_req {
	struct list_head list;
	int count;
	struct rpmh_request rpm_msgs[];
};

static struct rpmh_ctrlr *get_rpmh_ctrlr(const struct device *dev)
{
	struct rsc_drv *drv = dev_get_drvdata(dev->parent);

	return &drv->client;
}

void rpmh_tx_done(const struct tcs_request *msg)
{
	struct rpmh_request *rpm_msg = container_of(msg, struct rpmh_request,
						    msg);
	struct completion *compl = rpm_msg->completion;
	bool free = rpm_msg->needs_free;

	if (!compl)
		goto exit;

	/* Signal the blocking thread we are done */
	complete(compl);

exit:
	if (free)
		kfree(rpm_msg);
}

static struct cache_req *__find_req(struct rpmh_ctrlr *ctrlr, u32 addr)
{
	struct cache_req *p, *req = NULL;

	list_for_each_entry(p, &ctrlr->cache, list) {
		if (p->addr == addr) {
			req = p;
			break;
		}
	}

	return req;
}

static struct cache_req *cache_rpm_request(struct rpmh_ctrlr *ctrlr,
					   enum rpmh_state state,
					   struct tcs_cmd *cmd)
{
	struct cache_req *req;
	unsigned long flags;
	u32 old_sleep_val, old_wake_val;

	spin_lock_irqsave(&ctrlr->cache_lock, flags);
	req = __find_req(ctrlr, cmd->addr);
	if (req)
		goto existing;

	req = kzalloc(sizeof(*req), GFP_ATOMIC);
	if (!req) {
		req = ERR_PTR(-ENOMEM);
		goto unlock;
	}

	req->addr = cmd->addr;
	req->sleep_val = req->wake_val = UINT_MAX;
	list_add_tail(&req->list, &ctrlr->cache);

existing:
	old_sleep_val = req->sleep_val;
	old_wake_val = req->wake_val;

	switch (state) {
	case RPMH_ACTIVE_ONLY_STATE:
	case RPMH_WAKE_ONLY_STATE:
		req->wake_val = cmd->data;
		break;
	case RPMH_SLEEP_STATE:
		req->sleep_val = cmd->data;
		break;
	}

	ctrlr->dirty |= (req->sleep_val != old_sleep_val ||
			 req->wake_val != old_wake_val) &&
			 req->sleep_val != UINT_MAX &&
			 req->wake_val != UINT_MAX;

unlock:
	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);

	return req;
}

/**
 * __rpmh_write: Cache and send the RPMH request
 *
 * @dev: The device making the request
 * @state: Active/Sleep request type
 * @rpm_msg: The data that needs to be sent (cmds).
 *
 * Cache the RPMH request and send if the state is ACTIVE_ONLY.
 * SLEEP/WAKE_ONLY requests are not sent to the controller at
 * this time. Use rpmh_flush() to send them to the controller.
 */
static int __rpmh_write(const struct device *dev, enum rpmh_state state,
			struct rpmh_request *rpm_msg)
{
	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
	int ret = -EINVAL;
	struct cache_req *req;
	int i;

	/* Cache the request in our store and link the payload */
	for (i = 0; i < rpm_msg->msg.num_cmds; i++) {
		req = cache_rpm_request(ctrlr, state, &rpm_msg->msg.cmds[i]);
		if (IS_ERR(req))
			return PTR_ERR(req);
	}

	if (state == RPMH_ACTIVE_ONLY_STATE) {
		ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msg->msg);
	} else {
		/* Clean up our call by spoofing tx_done */
		ret = 0;
		rpmh_tx_done(&rpm_msg->msg);
	}

	return ret;
}

static int __fill_rpmh_msg(struct rpmh_request *req, enum rpmh_state state,
		const struct tcs_cmd *cmd, u32 n)
{
	if (!cmd || !n || n > MAX_RPMH_PAYLOAD)
		return -EINVAL;

	memcpy(req->cmd, cmd, n * sizeof(*cmd));

	req->msg.state = state;
	req->msg.cmds = req->cmd;
	req->msg.num_cmds = n;

	return 0;
}

/**
 * rpmh_write_async: Write a set of RPMH commands
 *
 * @dev: The device making the request
 * @state: Active/sleep set
 * @cmd: The payload data
 * @n: The number of elements in payload
 *
 * Write a set of RPMH commands, the order of commands is maintained
 * and will be sent as a single shot.
 */
int rpmh_write_async(const struct device *dev, enum rpmh_state state,
		     const struct tcs_cmd *cmd, u32 n)
{
	struct rpmh_request *rpm_msg;
	int ret;

	rpm_msg = kzalloc(sizeof(*rpm_msg), GFP_ATOMIC);
	if (!rpm_msg)
		return -ENOMEM;
	rpm_msg->needs_free = true;

	ret = __fill_rpmh_msg(rpm_msg, state, cmd, n);
	if (ret) {
		kfree(rpm_msg);
		return ret;
	}

	return __rpmh_write(dev, state, rpm_msg);
}
EXPORT_SYMBOL_GPL(rpmh_write_async);

/**
 * rpmh_write: Write a set of RPMH commands and block until response
 *
 * @dev: The device making the request
 * @state: Active/sleep set
 * @cmd: The payload data
 * @n: The number of elements in @cmd
 *
 * May sleep. Do not call from atomic contexts.
 */
int rpmh_write(const struct device *dev, enum rpmh_state state,
	       const struct tcs_cmd *cmd, u32 n)
{
	DECLARE_COMPLETION_ONSTACK(compl);
	DEFINE_RPMH_MSG_ONSTACK(dev, state, &compl, rpm_msg);
	int ret;

	ret = __fill_rpmh_msg(&rpm_msg, state, cmd, n);
	if (ret)
		return ret;

	ret = __rpmh_write(dev, state, &rpm_msg);
	if (ret)
		return ret;

	ret = wait_for_completion_timeout(&compl, RPMH_TIMEOUT_MS);
	WARN_ON(!ret);
	return (ret > 0) ? 0 : -ETIMEDOUT;
}
EXPORT_SYMBOL_GPL(rpmh_write);

static void cache_batch(struct rpmh_ctrlr *ctrlr, struct batch_cache_req *req)
{
	unsigned long flags;

	spin_lock_irqsave(&ctrlr->cache_lock, flags);
	list_add_tail(&req->list, &ctrlr->batch_cache);
	ctrlr->dirty = true;
	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
}

static int flush_batch(struct rpmh_ctrlr *ctrlr)
{
	struct batch_cache_req *req;
	const struct rpmh_request *rpm_msg;
	int ret = 0;
	int i;

	/* Send Sleep/Wake requests to the controller, expect no response */
	list_for_each_entry(req, &ctrlr->batch_cache, list) {
		for (i = 0; i < req->count; i++) {
			rpm_msg = req->rpm_msgs + i;
			ret = rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr),
						       &rpm_msg->msg);
			if (ret)
				break;
		}
	}

	return ret;
}

/**
 * rpmh_write_batch: Write multiple sets of RPMH commands and wait for the
 * batch to finish.
 *
 * @dev: the device making the request
 * @state: Active/sleep set
 * @cmd: The payload data
 * @n: The array of count of elements in each batch, 0 terminated.
 *
 * Write a request to the RSC controller without caching. If the request
 * state is ACTIVE, then the requests are treated as completion request
 * and sent to the controller immediately. The function waits until all the
 * commands are complete. If the request was to SLEEP or WAKE_ONLY, then the
 * request is sent as fire-n-forget and no ack is expected.
 *
 * May sleep. Do not call from atomic contexts for ACTIVE_ONLY requests.
 */
int rpmh_write_batch(const struct device *dev, enum rpmh_state state,
		     const struct tcs_cmd *cmd, u32 *n)
{
	struct batch_cache_req *req;
	struct rpmh_request *rpm_msgs;
	struct completion *compls;
	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
	unsigned long time_left;
	int count = 0;
	int ret, i;
	void *ptr;

	if (!cmd || !n)
		return -EINVAL;

	while (n[count] > 0)
		count++;
	if (!count)
		return -EINVAL;

	ptr = kzalloc(sizeof(*req) +
		      count * (sizeof(req->rpm_msgs[0]) + sizeof(*compls)),
		      GFP_ATOMIC);
	if (!ptr)
		return -ENOMEM;

	req = ptr;
	compls = ptr + sizeof(*req) + count * sizeof(*rpm_msgs);

	req->count = count;
	rpm_msgs = req->rpm_msgs;

	for (i = 0; i < count; i++) {
		__fill_rpmh_msg(rpm_msgs + i, state, cmd, n[i]);
		cmd += n[i];
	}

	if (state != RPMH_ACTIVE_ONLY_STATE) {
		cache_batch(ctrlr, req);
		return 0;
	}

	for (i = 0; i < count; i++) {
		struct completion *compl = &compls[i];

		init_completion(compl);
		rpm_msgs[i].completion = compl;
		ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msgs[i].msg);
		if (ret) {
			pr_err("Error(%d) sending RPMH message addr=%#x\n",
			       ret, rpm_msgs[i].msg.cmds[0].addr);
			break;
		}
	}

	time_left = RPMH_TIMEOUT_MS;
	while (i--) {
		time_left = wait_for_completion_timeout(&compls[i], time_left);
		if (!time_left) {
			/*
			 * Better hope they never finish because they'll signal
			 * the completion that we're going to free once
			 * we've returned from this function.
			 */
			WARN_ON(1);
			ret = -ETIMEDOUT;
			goto exit;
		}
	}

exit:
	kfree(ptr);

	return ret;
}
EXPORT_SYMBOL_GPL(rpmh_write_batch);

static int is_req_valid(struct cache_req *req)
{
	return (req->sleep_val != UINT_MAX &&
		req->wake_val != UINT_MAX &&
		req->sleep_val != req->wake_val);
}

static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state,
		       u32 addr, u32 data)
{
	DEFINE_RPMH_MSG_ONSTACK(NULL, state, NULL, rpm_msg);

	/* Wake sets are always complete and sleep sets are not */
	rpm_msg.msg.wait_for_compl = (state == RPMH_WAKE_ONLY_STATE);
	rpm_msg.cmd[0].addr = addr;
	rpm_msg.cmd[0].data = data;
	rpm_msg.msg.num_cmds = 1;

	return rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), &rpm_msg.msg);
}

/**
 * rpmh_flush() - Flushes the buffered sleep and wake sets to TCSes
 *
 * @ctrlr: Controller making request to flush cached data
 *
 * Return:
 * * 0          - Success
 * * Error code - Otherwise
 */
int rpmh_flush(struct rpmh_ctrlr *ctrlr)
{
	struct cache_req *p;
	int ret = 0;

	lockdep_assert_irqs_disabled();

	/*
	 * Currently rpmh_flush() is only called when we think we're running
	 * on the last processor.  If the lock is busy it means another
	 * processor is up and it's better to abort than spin.
	 */
	if (!spin_trylock(&ctrlr->cache_lock))
		return -EBUSY;

	if (!ctrlr->dirty) {
		pr_debug("Skipping flush, TCS has latest data.\n");
		goto write_next_wakeup;
	}

	/* Invalidate the TCSes first to avoid stale data */
	rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr));

	/* First flush the cached batch requests */
	ret = flush_batch(ctrlr);
	if (ret)
		goto exit;

	list_for_each_entry(p, &ctrlr->cache, list) {
		if (!is_req_valid(p)) {
			pr_debug("%s: skipping RPMH req: a:%#x s:%#x w:%#x",
				 __func__, p->addr, p->sleep_val, p->wake_val);
			continue;
		}
		ret = send_single(ctrlr, RPMH_SLEEP_STATE, p->addr,
				  p->sleep_val);
		if (ret)
			goto exit;
		ret = send_single(ctrlr, RPMH_WAKE_ONLY_STATE, p->addr,
				  p->wake_val);
		if (ret)
			goto exit;
	}

	ctrlr->dirty = false;

write_next_wakeup:
	rpmh_rsc_write_next_wakeup(ctrlr_to_drv(ctrlr));
exit:
	spin_unlock(&ctrlr->cache_lock);
	return ret;
}

/**
 * rpmh_invalidate: Invalidate sleep and wake sets in batch_cache
 *
 * @dev: The device making the request
 *
 * Invalidate the sleep and wake values in batch_cache.
 */
void rpmh_invalidate(const struct device *dev)
{
	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
	struct batch_cache_req *req, *tmp;
	unsigned long flags;

	spin_lock_irqsave(&ctrlr->cache_lock, flags);
	list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list)
		kfree(req);
	INIT_LIST_HEAD(&ctrlr->batch_cache);
	ctrlr->dirty = true;
	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
}
EXPORT_SYMBOL_GPL(rpmh_invalidate);
Commit	Line	Data
c1038456 LI	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
	4	*/
	5
	6	#include <linux/atomic.h>
	7	#include <linux/bug.h>
	8	#include <linux/interrupt.h>
	9	#include <linux/jiffies.h>
	10	#include <linux/kernel.h>
600513df	11	#include <linux/list.h>
985427f9	12	#include <linux/lockdep.h>
c1038456 LI	13	#include <linux/module.h>
	14	#include <linux/of.h>
	15	#include <linux/platform_device.h>
	16	#include <linux/slab.h>
600513df	17	#include <linux/spinlock.h>
c1038456 LI	18	#include <linux/types.h>
	19	#include <linux/wait.h>
	20
	21	#include <soc/qcom/rpmh.h>
	22
	23	#include "rpmh-internal.h"
	24
	25	#define RPMH_TIMEOUT_MS msecs_to_jiffies(10000)
	26
aff9cc08	27	#define DEFINE_RPMH_MSG_ONSTACK(device, s, q, name) \
c1038456 LI	28	struct rpmh_request name = { \
	29	.msg = { \
	30	.state = s, \
	31	.cmds = name.cmd, \
	32	.num_cmds = 0, \
	33	.wait_for_compl = true, \
	34	}, \
	35	.cmd = { { 0 } }, \
	36	.completion = q, \
aff9cc08	37	.dev = device, \
564b5e24	38	.needs_free = false, \
c1038456 LI	39	}
	40
	41	#define ctrlr_to_drv(ctrlr) container_of(ctrlr, struct rsc_drv, client)
	42
600513df LI	43	/**
	44	* struct cache_req: the request object for caching
	45	*
	46	* @addr: the address of the resource
	47	* @sleep_val: the sleep vote
	48	* @wake_val: the wake vote
	49	* @list: linked list obj
	50	*/
	51	struct cache_req {
	52	u32 addr;
	53	u32 sleep_val;
	54	u32 wake_val;
	55	struct list_head list;
	56	};
	57
c8790cb6 LI	58	/**
	59	* struct batch_cache_req - An entry in our batch catch
	60	*
	61	* @list: linked list obj
	62	* @count: number of messages
	63	* @rpm_msgs: the messages
	64	*/
	65
	66	struct batch_cache_req {
	67	struct list_head list;
	68	int count;
	69	struct rpmh_request rpm_msgs[];
	70	};
	71
c1038456 LI	72	static struct rpmh_ctrlr get_rpmh_ctrlr(const struct device dev)
	73	{
	74	struct rsc_drv *drv = dev_get_drvdata(dev->parent);
	75
	76	return &drv->client;
	77	}
	78
323dc2dc	79	void rpmh_tx_done(const struct tcs_request *msg)
c1038456 LI	80	{
	81	struct rpmh_request *rpm_msg = container_of(msg, struct rpmh_request,
	82	msg);
	83	struct completion *compl = rpm_msg->completion;
baef1c90	84	bool free = rpm_msg->needs_free;
c1038456	85
c8790cb6 LI	86	if (!compl)
	87	goto exit;
	88
c1038456	89	/* Signal the blocking thread we are done */
c8790cb6	90	complete(compl);
564b5e24	91
c8790cb6	92	exit:
baef1c90	93	if (free)
564b5e24	94	kfree(rpm_msg);
c1038456 LI	95	}
c1038456 LI	96
600513df LI	97	static struct cache_req __find_req(struct rpmh_ctrlr ctrlr, u32 addr)
	98	{
	99	struct cache_req p, req = NULL;
	100
	101	list_for_each_entry(p, &ctrlr->cache, list) {
	102	if (p->addr == addr) {
	103	req = p;
	104	break;
	105	}
	106	}
	107
	108	return req;
	109	}
	110
	111	static struct cache_req cache_rpm_request(struct rpmh_ctrlr ctrlr,
	112	enum rpmh_state state,
	113	struct tcs_cmd *cmd)
	114	{
	115	struct cache_req *req;
	116	unsigned long flags;
bb700067	117	u32 old_sleep_val, old_wake_val;
600513df LI	118
	119	spin_lock_irqsave(&ctrlr->cache_lock, flags);
	120	req = __find_req(ctrlr, cmd->addr);
	121	if (req)
	122	goto existing;
	123
	124	req = kzalloc(sizeof(*req), GFP_ATOMIC);
	125	if (!req) {
	126	req = ERR_PTR(-ENOMEM);
	127	goto unlock;
	128	}
	129
	130	req->addr = cmd->addr;
	131	req->sleep_val = req->wake_val = UINT_MAX;
600513df LI	132	list_add_tail(&req->list, &ctrlr->cache);
	133
	134	existing:
bb700067 MS	135	old_sleep_val = req->sleep_val;
	136	old_wake_val = req->wake_val;
	137
600513df LI	138	switch (state) {
600513df LI	139	case RPMH_ACTIVE_ONLY_STATE:
600513df LI	140	case RPMH_WAKE_ONLY_STATE:
	141	req->wake_val = cmd->data;
	142	break;
	143	case RPMH_SLEEP_STATE:
	144	req->sleep_val = cmd->data;
	145	break;
600513df LI	146	}
600513df LI	147
35bb4b22 DA	148	ctrlr->dirty \|= (req->sleep_val != old_sleep_val \|\|
	149	req->wake_val != old_wake_val) &&
	150	req->sleep_val != UINT_MAX &&
	151	req->wake_val != UINT_MAX;
bb700067	152
600513df LI	153	unlock:
	154	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
	155
	156	return req;
	157	}
	158
c1038456	159	/**
600513df	160	* __rpmh_write: Cache and send the RPMH request
c1038456 LI	161	*
	162	* @dev: The device making the request
	163	* @state: Active/Sleep request type
	164	* @rpm_msg: The data that needs to be sent (cmds).
600513df LI	165	*
	166	* Cache the RPMH request and send if the state is ACTIVE_ONLY.
	167	* SLEEP/WAKE_ONLY requests are not sent to the controller at
	168	* this time. Use rpmh_flush() to send them to the controller.
c1038456 LI	169	*/
	170	static int __rpmh_write(const struct device *dev, enum rpmh_state state,
	171	struct rpmh_request *rpm_msg)
	172	{
	173	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
600513df LI	174	int ret = -EINVAL;
	175	struct cache_req *req;
	176	int i;
c1038456	177
600513df LI	178	/* Cache the request in our store and link the payload */
	179	for (i = 0; i < rpm_msg->msg.num_cmds; i++) {
	180	req = cache_rpm_request(ctrlr, state, &rpm_msg->msg.cmds[i]);
	181	if (IS_ERR(req))
	182	return PTR_ERR(req);
	183	}
	184
600513df	185	if (state == RPMH_ACTIVE_ONLY_STATE) {
600513df LI	186	ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msg->msg);
600513df LI	187	} else {
600513df	188	/* Clean up our call by spoofing tx_done */
73002419	189	ret = 0;
323dc2dc	190	rpmh_tx_done(&rpm_msg->msg);
600513df	191	}
c1038456	192
600513df	193	return ret;
c1038456 LI	194	}
c1038456 LI	195
564b5e24 LI	196	static int __fill_rpmh_msg(struct rpmh_request *req, enum rpmh_state state,
	197	const struct tcs_cmd *cmd, u32 n)
	198	{
	199	if (!cmd \|\| !n \|\| n > MAX_RPMH_PAYLOAD)
	200	return -EINVAL;
	201
	202	memcpy(req->cmd, cmd, n * sizeof(*cmd));
	203
	204	req->msg.state = state;
	205	req->msg.cmds = req->cmd;
	206	req->msg.num_cmds = n;
	207
	208	return 0;
	209	}
	210
	211	/**
	212	* rpmh_write_async: Write a set of RPMH commands
	213	*
	214	* @dev: The device making the request
	215	* @state: Active/sleep set
	216	* @cmd: The payload data
	217	* @n: The number of elements in payload
	218	*
	219	* Write a set of RPMH commands, the order of commands is maintained
	220	* and will be sent as a single shot.
	221	*/
	222	int rpmh_write_async(const struct device *dev, enum rpmh_state state,
	223	const struct tcs_cmd *cmd, u32 n)
	224	{
	225	struct rpmh_request *rpm_msg;
	226	int ret;
	227
	228	rpm_msg = kzalloc(sizeof(*rpm_msg), GFP_ATOMIC);
	229	if (!rpm_msg)
	230	return -ENOMEM;
	231	rpm_msg->needs_free = true;
	232
	233	ret = __fill_rpmh_msg(rpm_msg, state, cmd, n);
	234	if (ret) {
	235	kfree(rpm_msg);
	236	return ret;
	237	}
	238
	239	return __rpmh_write(dev, state, rpm_msg);
	240	}
9b09c0f2	241	EXPORT_SYMBOL_GPL(rpmh_write_async);
564b5e24	242
c1038456 LI	243	/**
	244	* rpmh_write: Write a set of RPMH commands and block until response
	245	*
1894b78e	246	* @dev: The device making the request
c1038456 LI	247	* @state: Active/sleep set
	248	* @cmd: The payload data
	249	* @n: The number of elements in @cmd
	250	*
	251	* May sleep. Do not call from atomic contexts.
	252	*/
	253	int rpmh_write(const struct device *dev, enum rpmh_state state,
	254	const struct tcs_cmd *cmd, u32 n)
	255	{
	256	DECLARE_COMPLETION_ONSTACK(compl);
	257	DEFINE_RPMH_MSG_ONSTACK(dev, state, &compl, rpm_msg);
	258	int ret;
	259
0924dad5 MS	260	ret = __fill_rpmh_msg(&rpm_msg, state, cmd, n);
	261	if (ret)
	262	return ret;
c1038456 LI	263
	264	ret = __rpmh_write(dev, state, &rpm_msg);
	265	if (ret)
	266	return ret;
	267
	268	ret = wait_for_completion_timeout(&compl, RPMH_TIMEOUT_MS);
	269	WARN_ON(!ret);
	270	return (ret > 0) ? 0 : -ETIMEDOUT;
	271	}
9b09c0f2	272	EXPORT_SYMBOL_GPL(rpmh_write);
600513df	273
c8790cb6 LI	274	static void cache_batch(struct rpmh_ctrlr ctrlr, struct batch_cache_req req)
	275	{
	276	unsigned long flags;
	277
	278	spin_lock_irqsave(&ctrlr->cache_lock, flags);
	279	list_add_tail(&req->list, &ctrlr->batch_cache);
bb700067	280	ctrlr->dirty = true;
c8790cb6 LI	281	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
	282	}
	283
	284	static int flush_batch(struct rpmh_ctrlr *ctrlr)
	285	{
	286	struct batch_cache_req *req;
	287	const struct rpmh_request *rpm_msg;
c8790cb6 LI	288	int ret = 0;
	289	int i;
	290
	291	/* Send Sleep/Wake requests to the controller, expect no response */
c8790cb6 LI	292	list_for_each_entry(req, &ctrlr->batch_cache, list) {
	293	for (i = 0; i < req->count; i++) {
	294	rpm_msg = req->rpm_msgs + i;
	295	ret = rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr),
	296	&rpm_msg->msg);
	297	if (ret)
	298	break;
	299	}
	300	}
c8790cb6 LI	301
	302	return ret;
	303	}
	304
c8790cb6 LI	305	/**
	306	* rpmh_write_batch: Write multiple sets of RPMH commands and wait for the
	307	* batch to finish.
	308	*
	309	* @dev: the device making the request
	310	* @state: Active/sleep set
	311	* @cmd: The payload data
	312	* @n: The array of count of elements in each batch, 0 terminated.
	313	*
	314	* Write a request to the RSC controller without caching. If the request
	315	* state is ACTIVE, then the requests are treated as completion request
	316	* and sent to the controller immediately. The function waits until all the
	317	* commands are complete. If the request was to SLEEP or WAKE_ONLY, then the
	318	* request is sent as fire-n-forget and no ack is expected.
	319	*
	320	* May sleep. Do not call from atomic contexts for ACTIVE_ONLY requests.
	321	*/
	322	int rpmh_write_batch(const struct device *dev, enum rpmh_state state,
	323	const struct tcs_cmd cmd, u32 n)
	324	{
	325	struct batch_cache_req *req;
	326	struct rpmh_request *rpm_msgs;
baef1c90	327	struct completion *compls;
c8790cb6 LI	328	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
	329	unsigned long time_left;
	330	int count = 0;
baef1c90 SB	331	int ret, i;
baef1c90 SB	332	void *ptr;
c8790cb6 LI	333
	334	if (!cmd \|\| !n)
	335	return -EINVAL;
	336
	337	while (n[count] > 0)
	338	count++;
	339	if (!count)
	340	return -EINVAL;
	341
baef1c90 SB	342	ptr = kzalloc(sizeof(*req) +
baef1c90 SB	343	count * (sizeof(req->rpm_msgs[0]) + sizeof(*compls)),
c8790cb6	344	GFP_ATOMIC);
baef1c90	345	if (!ptr)
c8790cb6	346	return -ENOMEM;
baef1c90 SB	347
	348	req = ptr;
	349	compls = ptr + sizeof(req) + count sizeof(*rpm_msgs);
	350
c8790cb6 LI	351	req->count = count;
	352	rpm_msgs = req->rpm_msgs;
	353
	354	for (i = 0; i < count; i++) {
	355	__fill_rpmh_msg(rpm_msgs + i, state, cmd, n[i]);
	356	cmd += n[i];
	357	}
	358
	359	if (state != RPMH_ACTIVE_ONLY_STATE) {
	360	cache_batch(ctrlr, req);
	361	return 0;
	362	}
	363
	364	for (i = 0; i < count; i++) {
baef1c90 SB	365	struct completion *compl = &compls[i];
	366
	367	init_completion(compl);
	368	rpm_msgs[i].completion = compl;
c8790cb6 LI	369	ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msgs[i].msg);
	370	if (ret) {
	371	pr_err("Error(%d) sending RPMH message addr=%#x\n",
	372	ret, rpm_msgs[i].msg.cmds[0].addr);
c8790cb6 LI	373	break;
	374	}
	375	}
	376
	377	time_left = RPMH_TIMEOUT_MS;
baef1c90 SB	378	while (i--) {
baef1c90 SB	379	time_left = wait_for_completion_timeout(&compls[i], time_left);
c8790cb6 LI	380	if (!time_left) {
	381	/*
	382	* Better hope they never finish because they'll signal
baef1c90 SB	383	* the completion that we're going to free once
baef1c90 SB	384	* we've returned from this function.
c8790cb6 LI	385	*/
	386	WARN_ON(1);
	387	ret = -ETIMEDOUT;
	388	goto exit;
	389	}
	390	}
	391
	392	exit:
baef1c90	393	kfree(ptr);
c8790cb6 LI	394
	395	return ret;
	396	}
9b09c0f2	397	EXPORT_SYMBOL_GPL(rpmh_write_batch);
c8790cb6	398
600513df LI	399	static int is_req_valid(struct cache_req *req)
	400	{
	401	return (req->sleep_val != UINT_MAX &&
	402	req->wake_val != UINT_MAX &&
	403	req->sleep_val != req->wake_val);
	404	}
	405
d5e20507	406	static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state,
600513df LI	407	u32 addr, u32 data)
600513df LI	408	{
d5e20507	409	DEFINE_RPMH_MSG_ONSTACK(NULL, state, NULL, rpm_msg);
600513df LI	410
	411	/* Wake sets are always complete and sleep sets are not */
	412	rpm_msg.msg.wait_for_compl = (state == RPMH_WAKE_ONLY_STATE);
	413	rpm_msg.cmd[0].addr = addr;
	414	rpm_msg.cmd[0].data = data;
	415	rpm_msg.msg.num_cmds = 1;
	416
	417	return rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), &rpm_msg.msg);
	418	}
	419
	420	/**
985427f9	421	* rpmh_flush() - Flushes the buffered sleep and wake sets to TCSes
600513df	422	*
985427f9	423	* @ctrlr: Controller making request to flush cached data
600513df	424	*
985427f9 MS	425	* Return:
985427f9 MS	426	* * 0 - Success
985427f9	427	* * Error code - Otherwise
600513df	428	*/
d5e20507	429	int rpmh_flush(struct rpmh_ctrlr *ctrlr)
600513df LI	430	{
600513df LI	431	struct cache_req *p;
d2a8cfc6	432	int ret = 0;
600513df	433
985427f9 MS	434	lockdep_assert_irqs_disabled();
985427f9 MS	435
d2a8cfc6 DA	436	/*
	437	* Currently rpmh_flush() is only called when we think we're running
	438	* on the last processor. If the lock is busy it means another
	439	* processor is up and it's better to abort than spin.
	440	*/
	441	if (!spin_trylock(&ctrlr->cache_lock))
	442	return -EBUSY;
	443
600513df LI	444	if (!ctrlr->dirty) {
600513df LI	445	pr_debug("Skipping flush, TCS has latest data.\n");
cccbe3e5	446	goto write_next_wakeup;
600513df LI	447	}
600513df LI	448
f5ac95f9	449	/* Invalidate the TCSes first to avoid stale data */
881808d0	450	rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr));
f5ac95f9	451
c8790cb6 LI	452	/* First flush the cached batch requests */
	453	ret = flush_batch(ctrlr);
	454	if (ret)
d2a8cfc6	455	goto exit;
c8790cb6	456
600513df LI	457	list_for_each_entry(p, &ctrlr->cache, list) {
	458	if (!is_req_valid(p)) {
	459	pr_debug("%s: skipping RPMH req: a:%#x s:%#x w:%#x",
	460	__func__, p->addr, p->sleep_val, p->wake_val);
	461	continue;
	462	}
d5e20507 MS	463	ret = send_single(ctrlr, RPMH_SLEEP_STATE, p->addr,
d5e20507 MS	464	p->sleep_val);
600513df	465	if (ret)
d2a8cfc6	466	goto exit;
d5e20507 MS	467	ret = send_single(ctrlr, RPMH_WAKE_ONLY_STATE, p->addr,
d5e20507 MS	468	p->wake_val);
600513df	469	if (ret)
d2a8cfc6	470	goto exit;
600513df LI	471	}
	472
	473	ctrlr->dirty = false;
	474
cccbe3e5 MS	475	write_next_wakeup:
cccbe3e5 MS	476	rpmh_rsc_write_next_wakeup(ctrlr_to_drv(ctrlr));
d2a8cfc6 DA	477	exit:
	478	spin_unlock(&ctrlr->cache_lock);
	479	return ret;
600513df	480	}
600513df LI	481
600513df LI	482	/**
f5ac95f9	483	* rpmh_invalidate: Invalidate sleep and wake sets in batch_cache
600513df LI	484	*
	485	* @dev: The device making the request
	486	*
f5ac95f9	487	* Invalidate the sleep and wake values in batch_cache.
600513df	488	*/
73edcd38	489	void rpmh_invalidate(const struct device *dev)
600513df LI	490	{
600513df LI	491	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
f5ac95f9 MS	492	struct batch_cache_req req, tmp;
f5ac95f9 MS	493	unsigned long flags;
600513df	494
f5ac95f9 MS	495	spin_lock_irqsave(&ctrlr->cache_lock, flags);
	496	list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list)
	497	kfree(req);
	498	INIT_LIST_HEAD(&ctrlr->batch_cache);
	499	ctrlr->dirty = true;
	500	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
600513df	501	}
9b09c0f2	502	EXPORT_SYMBOL_GPL(rpmh_invalidate);