[linux-2.6-block.git] / drivers / mmc / host / mmc_hsq.c

// SPDX-License-Identifier: GPL-2.0
/*
 *
 * MMC software queue support based on command queue interfaces
 *
 * Copyright (C) 2019 Linaro, Inc.
 * Author: Baolin Wang <baolin.wang@linaro.org>
 */

#include <linux/mmc/card.h>
#include <linux/mmc/host.h>
#include <linux/module.h>

#include "mmc_hsq.h"

static void mmc_hsq_retry_handler(struct work_struct *work)
{
	struct mmc_hsq *hsq = container_of(work, struct mmc_hsq, retry_work);
	struct mmc_host *mmc = hsq->mmc;

	mmc->ops->request(mmc, hsq->mrq);
}

static void mmc_hsq_pump_requests(struct mmc_hsq *hsq)
{
	struct mmc_host *mmc = hsq->mmc;
	struct hsq_slot *slot;
	unsigned long flags;
	int ret = 0;

	spin_lock_irqsave(&hsq->lock, flags);

	/* Make sure we are not already running a request now */
	if (hsq->mrq || hsq->recovery_halt) {
		spin_unlock_irqrestore(&hsq->lock, flags);
		return;
	}

	/* Make sure there are remain requests need to pump */
	if (!hsq->qcnt || !hsq->enabled) {
		spin_unlock_irqrestore(&hsq->lock, flags);
		return;
	}

	slot = &hsq->slot[hsq->next_tag];
	hsq->mrq = slot->mrq;
	hsq->qcnt--;

	spin_unlock_irqrestore(&hsq->lock, flags);

	if (mmc->ops->request_atomic)
		ret = mmc->ops->request_atomic(mmc, hsq->mrq);
	else
		mmc->ops->request(mmc, hsq->mrq);

	/*
	 * If returning BUSY from request_atomic(), which means the card
	 * may be busy now, and we should change to non-atomic context to
	 * try again for this unusual case, to avoid time-consuming operations
	 * in the atomic context.
	 *
	 * Note: we just give a warning for other error cases, since the host
	 * driver will handle them.
	 */
	if (ret == -EBUSY)
		schedule_work(&hsq->retry_work);
	else
		WARN_ON_ONCE(ret);
}

static void mmc_hsq_update_next_tag(struct mmc_hsq *hsq, int remains)
{
	int tag;

	/*
	 * If there are no remain requests in software queue, then set a invalid
	 * tag.
	 */
	if (!remains) {
		hsq->next_tag = HSQ_INVALID_TAG;
		hsq->tail_tag = HSQ_INVALID_TAG;
		return;
	}

	tag = hsq->tag_slot[hsq->next_tag];
	hsq->tag_slot[hsq->next_tag] = HSQ_INVALID_TAG;
	hsq->next_tag = tag;
}

static void mmc_hsq_post_request(struct mmc_hsq *hsq)
{
	unsigned long flags;
	int remains;

	spin_lock_irqsave(&hsq->lock, flags);

	remains = hsq->qcnt;
	hsq->mrq = NULL;

	/* Update the next available tag to be queued. */
	mmc_hsq_update_next_tag(hsq, remains);

	if (hsq->waiting_for_idle && !remains) {
		hsq->waiting_for_idle = false;
		wake_up(&hsq->wait_queue);
	}

	/* Do not pump new request in recovery mode. */
	if (hsq->recovery_halt) {
		spin_unlock_irqrestore(&hsq->lock, flags);
		return;
	}

	spin_unlock_irqrestore(&hsq->lock, flags);

	 /*
	  * Try to pump new request to host controller as fast as possible,
	  * after completing previous request.
	  */
	if (remains > 0)
		mmc_hsq_pump_requests(hsq);
}

/**
 * mmc_hsq_finalize_request - finalize one request if the request is done
 * @mmc: the host controller
 * @mrq: the request need to be finalized
 *
 * Return true if we finalized the corresponding request in software queue,
 * otherwise return false.
 */
bool mmc_hsq_finalize_request(struct mmc_host *mmc, struct mmc_request *mrq)
{
	struct mmc_hsq *hsq = mmc->cqe_private;
	unsigned long flags;

	spin_lock_irqsave(&hsq->lock, flags);

	if (!hsq->enabled || !hsq->mrq || hsq->mrq != mrq) {
		spin_unlock_irqrestore(&hsq->lock, flags);
		return false;
	}

	/*
	 * Clear current completed slot request to make a room for new request.
	 */
	hsq->slot[hsq->next_tag].mrq = NULL;

	spin_unlock_irqrestore(&hsq->lock, flags);

	mmc_cqe_request_done(mmc, hsq->mrq);

	mmc_hsq_post_request(hsq);

	return true;
}
EXPORT_SYMBOL_GPL(mmc_hsq_finalize_request);

static void mmc_hsq_recovery_start(struct mmc_host *mmc)
{
	struct mmc_hsq *hsq = mmc->cqe_private;
	unsigned long flags;

	spin_lock_irqsave(&hsq->lock, flags);

	hsq->recovery_halt = true;

	spin_unlock_irqrestore(&hsq->lock, flags);
}

static void mmc_hsq_recovery_finish(struct mmc_host *mmc)
{
	struct mmc_hsq *hsq = mmc->cqe_private;
	int remains;

	spin_lock_irq(&hsq->lock);

	hsq->recovery_halt = false;
	remains = hsq->qcnt;

	spin_unlock_irq(&hsq->lock);

	/*
	 * Try to pump new request if there are request pending in software
	 * queue after finishing recovery.
	 */
	if (remains > 0)
		mmc_hsq_pump_requests(hsq);
}

static int mmc_hsq_request(struct mmc_host *mmc, struct mmc_request *mrq)
{
	struct mmc_hsq *hsq = mmc->cqe_private;
	int tag = mrq->tag;

	spin_lock_irq(&hsq->lock);

	if (!hsq->enabled) {
		spin_unlock_irq(&hsq->lock);
		return -ESHUTDOWN;
	}

	/* Do not queue any new requests in recovery mode. */
	if (hsq->recovery_halt) {
		spin_unlock_irq(&hsq->lock);
		return -EBUSY;
	}

	hsq->slot[tag].mrq = mrq;

	/*
	 * Set the next tag as current request tag if no available
	 * next tag.
	 */
	if (hsq->next_tag == HSQ_INVALID_TAG) {
		hsq->next_tag = tag;
		hsq->tail_tag = tag;
		hsq->tag_slot[hsq->tail_tag] = HSQ_INVALID_TAG;
	} else {
		hsq->tag_slot[hsq->tail_tag] = tag;
		hsq->tail_tag = tag;
	}

	hsq->qcnt++;

	spin_unlock_irq(&hsq->lock);

	mmc_hsq_pump_requests(hsq);

	return 0;
}

static void mmc_hsq_post_req(struct mmc_host *mmc, struct mmc_request *mrq)
{
	if (mmc->ops->post_req)
		mmc->ops->post_req(mmc, mrq, 0);
}

static bool mmc_hsq_queue_is_idle(struct mmc_hsq *hsq, int *ret)
{
	bool is_idle;

	spin_lock_irq(&hsq->lock);

	is_idle = (!hsq->mrq && !hsq->qcnt) ||
		hsq->recovery_halt;

	*ret = hsq->recovery_halt ? -EBUSY : 0;
	hsq->waiting_for_idle = !is_idle;

	spin_unlock_irq(&hsq->lock);

	return is_idle;
}

static int mmc_hsq_wait_for_idle(struct mmc_host *mmc)
{
	struct mmc_hsq *hsq = mmc->cqe_private;
	int ret;

	wait_event(hsq->wait_queue,
		   mmc_hsq_queue_is_idle(hsq, &ret));

	return ret;
}

static void mmc_hsq_disable(struct mmc_host *mmc)
{
	struct mmc_hsq *hsq = mmc->cqe_private;
	u32 timeout = 500;
	int ret;

	spin_lock_irq(&hsq->lock);

	if (!hsq->enabled) {
		spin_unlock_irq(&hsq->lock);
		return;
	}

	spin_unlock_irq(&hsq->lock);

	ret = wait_event_timeout(hsq->wait_queue,
				 mmc_hsq_queue_is_idle(hsq, &ret),
				 msecs_to_jiffies(timeout));
	if (ret == 0) {
		pr_warn("could not stop mmc software queue\n");
		return;
	}

	spin_lock_irq(&hsq->lock);

	hsq->enabled = false;

	spin_unlock_irq(&hsq->lock);
}

static int mmc_hsq_enable(struct mmc_host *mmc, struct mmc_card *card)
{
	struct mmc_hsq *hsq = mmc->cqe_private;

	spin_lock_irq(&hsq->lock);

	if (hsq->enabled) {
		spin_unlock_irq(&hsq->lock);
		return -EBUSY;
	}

	hsq->enabled = true;

	spin_unlock_irq(&hsq->lock);

	return 0;
}

static const struct mmc_cqe_ops mmc_hsq_ops = {
	.cqe_enable = mmc_hsq_enable,
	.cqe_disable = mmc_hsq_disable,
	.cqe_request = mmc_hsq_request,
	.cqe_post_req = mmc_hsq_post_req,
	.cqe_wait_for_idle = mmc_hsq_wait_for_idle,
	.cqe_recovery_start = mmc_hsq_recovery_start,
	.cqe_recovery_finish = mmc_hsq_recovery_finish,
};

int mmc_hsq_init(struct mmc_hsq *hsq, struct mmc_host *mmc)
{
	int i;
	hsq->num_slots = HSQ_NUM_SLOTS;
	hsq->next_tag = HSQ_INVALID_TAG;
	hsq->tail_tag = HSQ_INVALID_TAG;

	hsq->slot = devm_kcalloc(mmc_dev(mmc), hsq->num_slots,
				 sizeof(struct hsq_slot), GFP_KERNEL);
	if (!hsq->slot)
		return -ENOMEM;

	hsq->mmc = mmc;
	hsq->mmc->cqe_private = hsq;
	mmc->cqe_ops = &mmc_hsq_ops;

	for (i = 0; i < HSQ_NUM_SLOTS; i++)
		hsq->tag_slot[i] = HSQ_INVALID_TAG;

	INIT_WORK(&hsq->retry_work, mmc_hsq_retry_handler);
	spin_lock_init(&hsq->lock);
	init_waitqueue_head(&hsq->wait_queue);

	return 0;
}
EXPORT_SYMBOL_GPL(mmc_hsq_init);

void mmc_hsq_suspend(struct mmc_host *mmc)
{
	mmc_hsq_disable(mmc);
}
EXPORT_SYMBOL_GPL(mmc_hsq_suspend);

int mmc_hsq_resume(struct mmc_host *mmc)
{
	return mmc_hsq_enable(mmc, NULL);
}
EXPORT_SYMBOL_GPL(mmc_hsq_resume);

MODULE_DESCRIPTION("MMC Host Software Queue support");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
511ce378 BW	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	*
	4	* MMC software queue support based on command queue interfaces
	5	*
	6	* Copyright (C) 2019 Linaro, Inc.
	7	* Author: Baolin Wang <baolin.wang@linaro.org>
	8	*/
	9
	10	#include <linux/mmc/card.h>
	11	#include <linux/mmc/host.h>
d1709abb	12	#include <linux/module.h>
511ce378 BW	13
	14	#include "mmc_hsq.h"
	15
6db96e58 BW	16	static void mmc_hsq_retry_handler(struct work_struct *work)
	17	{
	18	struct mmc_hsq *hsq = container_of(work, struct mmc_hsq, retry_work);
	19	struct mmc_host *mmc = hsq->mmc;
	20
	21	mmc->ops->request(mmc, hsq->mrq);
	22	}
	23
511ce378 BW	24	static void mmc_hsq_pump_requests(struct mmc_hsq *hsq)
	25	{
	26	struct mmc_host *mmc = hsq->mmc;
	27	struct hsq_slot *slot;
	28	unsigned long flags;
6db96e58	29	int ret = 0;
511ce378 BW	30
	31	spin_lock_irqsave(&hsq->lock, flags);
	32
	33	/* Make sure we are not already running a request now */
e7afa79a	34	if (hsq->mrq \|\| hsq->recovery_halt) {
511ce378 BW	35	spin_unlock_irqrestore(&hsq->lock, flags);
	36	return;
	37	}
	38
	39	/* Make sure there are remain requests need to pump */
	40	if (!hsq->qcnt \|\| !hsq->enabled) {
	41	spin_unlock_irqrestore(&hsq->lock, flags);
	42	return;
	43	}
	44
	45	slot = &hsq->slot[hsq->next_tag];
	46	hsq->mrq = slot->mrq;
	47	hsq->qcnt--;
	48
	49	spin_unlock_irqrestore(&hsq->lock, flags);
	50
6db96e58 BW	51	if (mmc->ops->request_atomic)
	52	ret = mmc->ops->request_atomic(mmc, hsq->mrq);
	53	else
	54	mmc->ops->request(mmc, hsq->mrq);
	55
	56	/*
	57	* If returning BUSY from request_atomic(), which means the card
	58	* may be busy now, and we should change to non-atomic context to
	59	* try again for this unusual case, to avoid time-consuming operations
	60	* in the atomic context.
	61	*
	62	* Note: we just give a warning for other error cases, since the host
	63	* driver will handle them.
	64	*/
	65	if (ret == -EBUSY)
	66	schedule_work(&hsq->retry_work);
	67	else
	68	WARN_ON_ONCE(ret);
511ce378 BW	69	}
	70
	71	static void mmc_hsq_update_next_tag(struct mmc_hsq *hsq, int remains)
	72	{
511ce378 BW	73	int tag;
	74
	75	/*
	76	* If there are no remain requests in software queue, then set a invalid
	77	* tag.
	78	*/
	79	if (!remains) {
	80	hsq->next_tag = HSQ_INVALID_TAG;
e026a3f9	81	hsq->tail_tag = HSQ_INVALID_TAG;
511ce378 BW	82	return;
	83	}
	84
e026a3f9 MW	85	tag = hsq->tag_slot[hsq->next_tag];
e026a3f9 MW	86	hsq->tag_slot[hsq->next_tag] = HSQ_INVALID_TAG;
511ce378 BW	87	hsq->next_tag = tag;
	88	}
	89
	90	static void mmc_hsq_post_request(struct mmc_hsq *hsq)
	91	{
	92	unsigned long flags;
	93	int remains;
	94
	95	spin_lock_irqsave(&hsq->lock, flags);
	96
	97	remains = hsq->qcnt;
	98	hsq->mrq = NULL;
	99
	100	/* Update the next available tag to be queued. */
	101	mmc_hsq_update_next_tag(hsq, remains);
	102
	103	if (hsq->waiting_for_idle && !remains) {
	104	hsq->waiting_for_idle = false;
	105	wake_up(&hsq->wait_queue);
	106	}
	107
	108	/* Do not pump new request in recovery mode. */
	109	if (hsq->recovery_halt) {
	110	spin_unlock_irqrestore(&hsq->lock, flags);
	111	return;
	112	}
	113
	114	spin_unlock_irqrestore(&hsq->lock, flags);
	115
	116	/*
	117	* Try to pump new request to host controller as fast as possible,
	118	* after completing previous request.
	119	*/
	120	if (remains > 0)
	121	mmc_hsq_pump_requests(hsq);
	122	}
	123
	124	/**
	125	* mmc_hsq_finalize_request - finalize one request if the request is done
	126	* @mmc: the host controller
	127	* @mrq: the request need to be finalized
	128	*
	129	* Return true if we finalized the corresponding request in software queue,
	130	* otherwise return false.
	131	*/
	132	bool mmc_hsq_finalize_request(struct mmc_host mmc, struct mmc_request mrq)
	133	{
	134	struct mmc_hsq *hsq = mmc->cqe_private;
	135	unsigned long flags;
	136
	137	spin_lock_irqsave(&hsq->lock, flags);
	138
	139	if (!hsq->enabled \|\| !hsq->mrq \|\| hsq->mrq != mrq) {
	140	spin_unlock_irqrestore(&hsq->lock, flags);
	141	return false;
	142	}
	143
	144	/*
	145	* Clear current completed slot request to make a room for new request.
	146	*/
	147	hsq->slot[hsq->next_tag].mrq = NULL;
	148
	149	spin_unlock_irqrestore(&hsq->lock, flags);
	150
151	mmc_cqe_request_done(mmc, hsq->mrq);
152
153	mmc_hsq_post_request(hsq);
154
155	return true;
156	}
157	EXPORT_SYMBOL_GPL(mmc_hsq_finalize_request);
158
159	static void mmc_hsq_recovery_start(struct mmc_host *mmc)
160	{
161	struct mmc_hsq *hsq = mmc->cqe_private;
162	unsigned long flags;
163
164	spin_lock_irqsave(&hsq->lock, flags);
165
166	hsq->recovery_halt = true;
167
168	spin_unlock_irqrestore(&hsq->lock, flags);
169	}
170
171	static void mmc_hsq_recovery_finish(struct mmc_host *mmc)
172	{
173	struct mmc_hsq *hsq = mmc->cqe_private;
174	int remains;
175
176	spin_lock_irq(&hsq->lock);
177
178	hsq->recovery_halt = false;
179	remains = hsq->qcnt;
180
181	spin_unlock_irq(&hsq->lock);
182
183	/*
184	* Try to pump new request if there are request pending in software
185	* queue after finishing recovery.
186	*/
187	if (remains > 0)
188	mmc_hsq_pump_requests(hsq);
189	}
190
191	static int mmc_hsq_request(struct mmc_host mmc, struct mmc_request mrq)
192	{
193	struct mmc_hsq *hsq = mmc->cqe_private;
194	int tag = mrq->tag;
195
196	spin_lock_irq(&hsq->lock);
197
198	if (!hsq->enabled) {
199	spin_unlock_irq(&hsq->lock);
200	return -ESHUTDOWN;
201	}
202
203	/* Do not queue any new requests in recovery mode. */
204	if (hsq->recovery_halt) {
205	spin_unlock_irq(&hsq->lock);
206	return -EBUSY;
207	}
208
209	hsq->slot[tag].mrq = mrq;
210
211	/*
212	* Set the next tag as current request tag if no available
213	* next tag.
214	*/
e026a3f9	215	if (hsq->next_tag == HSQ_INVALID_TAG) {
511ce378	216	hsq->next_tag = tag;
e026a3f9 MW	217	hsq->tail_tag = tag;
	218	hsq->tag_slot[hsq->tail_tag] = HSQ_INVALID_TAG;
	219	} else {
	220	hsq->tag_slot[hsq->tail_tag] = tag;
	221	hsq->tail_tag = tag;
	222	}
511ce378 BW	223
	224	hsq->qcnt++;
	225
	226	spin_unlock_irq(&hsq->lock);
	227
	228	mmc_hsq_pump_requests(hsq);
	229
	230	return 0;
	231	}
	232
	233	static void mmc_hsq_post_req(struct mmc_host mmc, struct mmc_request mrq)
	234	{
	235	if (mmc->ops->post_req)
	236	mmc->ops->post_req(mmc, mrq, 0);
	237	}
	238
	239	static bool mmc_hsq_queue_is_idle(struct mmc_hsq hsq, int ret)
	240	{
	241	bool is_idle;
	242
	243	spin_lock_irq(&hsq->lock);
	244
	245	is_idle = (!hsq->mrq && !hsq->qcnt) \|\|
	246	hsq->recovery_halt;
	247
	248	*ret = hsq->recovery_halt ? -EBUSY : 0;
	249	hsq->waiting_for_idle = !is_idle;
	250
	251	spin_unlock_irq(&hsq->lock);
	252
	253	return is_idle;
	254	}
	255
	256	static int mmc_hsq_wait_for_idle(struct mmc_host *mmc)
	257	{
	258	struct mmc_hsq *hsq = mmc->cqe_private;
	259	int ret;
	260
	261	wait_event(hsq->wait_queue,
	262	mmc_hsq_queue_is_idle(hsq, &ret));
	263
	264	return ret;
	265	}
	266
	267	static void mmc_hsq_disable(struct mmc_host *mmc)
	268	{
	269	struct mmc_hsq *hsq = mmc->cqe_private;
	270	u32 timeout = 500;
	271	int ret;
	272
	273	spin_lock_irq(&hsq->lock);
	274
	275	if (!hsq->enabled) {
	276	spin_unlock_irq(&hsq->lock);
	277	return;
	278	}
	279
	280	spin_unlock_irq(&hsq->lock);
	281
	282	ret = wait_event_timeout(hsq->wait_queue,
	283	mmc_hsq_queue_is_idle(hsq, &ret),
	284	msecs_to_jiffies(timeout));
	285	if (ret == 0) {
	286	pr_warn("could not stop mmc software queue\n");
287	return;
288	}
289
290	spin_lock_irq(&hsq->lock);
291
292	hsq->enabled = false;
293
294	spin_unlock_irq(&hsq->lock);
295	}
296
297	static int mmc_hsq_enable(struct mmc_host mmc, struct mmc_card card)
298	{
299	struct mmc_hsq *hsq = mmc->cqe_private;
300
301	spin_lock_irq(&hsq->lock);
302
303	if (hsq->enabled) {
304	spin_unlock_irq(&hsq->lock);
305	return -EBUSY;
306	}
307
308	hsq->enabled = true;
309
310	spin_unlock_irq(&hsq->lock);
311
312	return 0;
313	}
314
315	static const struct mmc_cqe_ops mmc_hsq_ops = {
316	.cqe_enable = mmc_hsq_enable,
317	.cqe_disable = mmc_hsq_disable,
318	.cqe_request = mmc_hsq_request,
319	.cqe_post_req = mmc_hsq_post_req,
320	.cqe_wait_for_idle = mmc_hsq_wait_for_idle,
321	.cqe_recovery_start = mmc_hsq_recovery_start,
322	.cqe_recovery_finish = mmc_hsq_recovery_finish,
323	};
324
325	int mmc_hsq_init(struct mmc_hsq hsq, struct mmc_host mmc)
326	{
e026a3f9	327	int i;
511ce378 BW	328	hsq->num_slots = HSQ_NUM_SLOTS;
511ce378 BW	329	hsq->next_tag = HSQ_INVALID_TAG;
e026a3f9	330	hsq->tail_tag = HSQ_INVALID_TAG;
511ce378 BW	331
	332	hsq->slot = devm_kcalloc(mmc_dev(mmc), hsq->num_slots,
	333	sizeof(struct hsq_slot), GFP_KERNEL);
	334	if (!hsq->slot)
	335	return -ENOMEM;
	336
	337	hsq->mmc = mmc;
	338	hsq->mmc->cqe_private = hsq;
	339	mmc->cqe_ops = &mmc_hsq_ops;
	340
e026a3f9 MW	341	for (i = 0; i < HSQ_NUM_SLOTS; i++)
	342	hsq->tag_slot[i] = HSQ_INVALID_TAG;
	343
6db96e58	344	INIT_WORK(&hsq->retry_work, mmc_hsq_retry_handler);
511ce378 BW	345	spin_lock_init(&hsq->lock);
	346	init_waitqueue_head(&hsq->wait_queue);
	347
	348	return 0;
	349	}
	350	EXPORT_SYMBOL_GPL(mmc_hsq_init);
	351
	352	void mmc_hsq_suspend(struct mmc_host *mmc)
	353	{
	354	mmc_hsq_disable(mmc);
	355	}
	356	EXPORT_SYMBOL_GPL(mmc_hsq_suspend);
	357
	358	int mmc_hsq_resume(struct mmc_host *mmc)
	359	{
	360	return mmc_hsq_enable(mmc, NULL);
	361	}
	362	EXPORT_SYMBOL_GPL(mmc_hsq_resume);
d1709abb BW	363
	364	MODULE_DESCRIPTION("MMC Host Software Queue support");
	365	MODULE_LICENSE("GPL v2");