[linux-block.git] / crypto / crypto_engine.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Handle async block request by crypto hardware engine.
 *
 * Copyright (C) 2016 Linaro, Inc.
 *
 * Author: Baolin Wang <baolin.wang@linaro.org>
 */

#include <linux/err.h>
#include <linux/delay.h>
#include <crypto/engine.h>
#include <uapi/linux/sched/types.h>
#include "internal.h"

#define CRYPTO_ENGINE_MAX_QLEN 10

/**
 * crypto_finalize_request - finalize one request if the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
static void crypto_finalize_request(struct crypto_engine *engine,
				    struct crypto_async_request *req, int err)
{
	unsigned long flags;
	bool finalize_req = false;
	int ret;
	struct crypto_engine_ctx *enginectx;

	/*
	 * If hardware cannot enqueue more requests
	 * and retry mechanism is not supported
	 * make sure we are completing the current request
	 */
	if (!engine->retry_support) {
		spin_lock_irqsave(&engine->queue_lock, flags);
		if (engine->cur_req == req) {
			finalize_req = true;
			engine->cur_req = NULL;
		}
		spin_unlock_irqrestore(&engine->queue_lock, flags);
	}

	if (finalize_req || engine->retry_support) {
		enginectx = crypto_tfm_ctx(req->tfm);
		if (enginectx->op.prepare_request &&
		    enginectx->op.unprepare_request) {
			ret = enginectx->op.unprepare_request(engine, req);
			if (ret)
				dev_err(engine->dev, "failed to unprepare request\n");
		}
	}
	req->complete(req, err);

	kthread_queue_work(engine->kworker, &engine->pump_requests);
}

/**
 * crypto_pump_requests - dequeue one request from engine queue to process
 * @engine: the hardware engine
 * @in_kthread: true if we are in the context of the request pump thread
 *
 * This function checks if there is any request in the engine queue that
 * needs processing and if so call out to the driver to initialize hardware
 * and handle each request.
 */
static void crypto_pump_requests(struct crypto_engine *engine,
				 bool in_kthread)
{
	struct crypto_async_request *async_req, *backlog;
	unsigned long flags;
	bool was_busy = false;
	int ret;
	struct crypto_engine_ctx *enginectx;

	spin_lock_irqsave(&engine->queue_lock, flags);

	/* Make sure we are not already running a request */
	if (!engine->retry_support && engine->cur_req)
		goto out;

	/* If another context is idling then defer */
	if (engine->idling) {
		kthread_queue_work(engine->kworker, &engine->pump_requests);
		goto out;
	}

	/* Check if the engine queue is idle */
	if (!crypto_queue_len(&engine->queue) || !engine->running) {
		if (!engine->busy)
			goto out;

		/* Only do teardown in the thread */
		if (!in_kthread) {
			kthread_queue_work(engine->kworker,
					   &engine->pump_requests);
			goto out;
		}

		engine->busy = false;
		engine->idling = true;
		spin_unlock_irqrestore(&engine->queue_lock, flags);

		if (engine->unprepare_crypt_hardware &&
		    engine->unprepare_crypt_hardware(engine))
			dev_err(engine->dev, "failed to unprepare crypt hardware\n");

		spin_lock_irqsave(&engine->queue_lock, flags);
		engine->idling = false;
		goto out;
	}

start_request:
	/* Get the fist request from the engine queue to handle */
	backlog = crypto_get_backlog(&engine->queue);
	async_req = crypto_dequeue_request(&engine->queue);
	if (!async_req)
		goto out;

	/*
	 * If hardware doesn't support the retry mechanism,
	 * keep track of the request we are processing now.
	 * We'll need it on completion (crypto_finalize_request).
	 */
	if (!engine->retry_support)
		engine->cur_req = async_req;

	if (backlog)
		backlog->complete(backlog, -EINPROGRESS);

	if (engine->busy)
		was_busy = true;
	else
		engine->busy = true;

	spin_unlock_irqrestore(&engine->queue_lock, flags);

	/* Until here we get the request need to be encrypted successfully */
	if (!was_busy && engine->prepare_crypt_hardware) {
		ret = engine->prepare_crypt_hardware(engine);
		if (ret) {
			dev_err(engine->dev, "failed to prepare crypt hardware\n");
			goto req_err_2;
		}
	}

	enginectx = crypto_tfm_ctx(async_req->tfm);

	if (enginectx->op.prepare_request) {
		ret = enginectx->op.prepare_request(engine, async_req);
		if (ret) {
			dev_err(engine->dev, "failed to prepare request: %d\n",
				ret);
			goto req_err_2;
		}
	}
	if (!enginectx->op.do_one_request) {
		dev_err(engine->dev, "failed to do request\n");
		ret = -EINVAL;
		goto req_err_1;
	}

	ret = enginectx->op.do_one_request(engine, async_req);

	/* Request unsuccessfully executed by hardware */
	if (ret < 0) {
		/*
		 * If hardware queue is full (-ENOSPC), requeue request
		 * regardless of backlog flag.
		 * If hardware throws any other error code,
		 * requeue only backlog requests.
		 * Otherwise, unprepare and complete the request.
		 */
		if (!engine->retry_support ||
		    ((ret != -ENOSPC) &&
		    !(async_req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) {
			dev_err(engine->dev,
				"Failed to do one request from queue: %d\n",
				ret);
			goto req_err_1;
		}
		/*
		 * If retry mechanism is supported,
		 * unprepare current request and
		 * enqueue it back into crypto-engine queue.
		 */
		if (enginectx->op.unprepare_request) {
			ret = enginectx->op.unprepare_request(engine,
							      async_req);
			if (ret)
				dev_err(engine->dev,
					"failed to unprepare request\n");
		}
		spin_lock_irqsave(&engine->queue_lock, flags);
		/*
		 * If hardware was unable to execute request, enqueue it
		 * back in front of crypto-engine queue, to keep the order
		 * of requests.
		 */
		crypto_enqueue_request_head(&engine->queue, async_req);

		kthread_queue_work(engine->kworker, &engine->pump_requests);
		goto out;
	}

	goto retry;

req_err_1:
	if (enginectx->op.unprepare_request) {
		ret = enginectx->op.unprepare_request(engine, async_req);
		if (ret)
			dev_err(engine->dev, "failed to unprepare request\n");
	}

req_err_2:
	async_req->complete(async_req, ret);

retry:
	/* If retry mechanism is supported, send new requests to engine */
	if (engine->retry_support) {
		spin_lock_irqsave(&engine->queue_lock, flags);
		goto start_request;
	}
	return;

out:
	spin_unlock_irqrestore(&engine->queue_lock, flags);

	/*
	 * Batch requests is possible only if
	 * hardware can enqueue multiple requests
	 */
	if (engine->do_batch_requests) {
		ret = engine->do_batch_requests(engine);
		if (ret)
			dev_err(engine->dev, "failed to do batch requests: %d\n",
				ret);
	}

	return;
}

static void crypto_pump_work(struct kthread_work *work)
{
	struct crypto_engine *engine =
		container_of(work, struct crypto_engine, pump_requests);

	crypto_pump_requests(engine, true);
}

/**
 * crypto_transfer_request - transfer the new request into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
static int crypto_transfer_request(struct crypto_engine *engine,
				   struct crypto_async_request *req,
				   bool need_pump)
{
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&engine->queue_lock, flags);

	if (!engine->running) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		return -ESHUTDOWN;
	}

	ret = crypto_enqueue_request(&engine->queue, req);

	if (!engine->busy && need_pump)
		kthread_queue_work(engine->kworker, &engine->pump_requests);

	spin_unlock_irqrestore(&engine->queue_lock, flags);
	return ret;
}

/**
 * crypto_transfer_request_to_engine - transfer one request to list
 * into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
					     struct crypto_async_request *req)
{
	return crypto_transfer_request(engine, req, true);
}

/**
 * crypto_transfer_aead_request_to_engine - transfer one aead_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
					   struct aead_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);

/**
 * crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
					       struct akcipher_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);

/**
 * crypto_transfer_hash_request_to_engine - transfer one ahash_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
					   struct ahash_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);

/**
 * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
					       struct skcipher_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);

/**
 * crypto_finalize_aead_request - finalize one aead_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_aead_request(struct crypto_engine *engine,
				  struct aead_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);

/**
 * crypto_finalize_akcipher_request - finalize one akcipher_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_akcipher_request(struct crypto_engine *engine,
				      struct akcipher_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);

/**
 * crypto_finalize_hash_request - finalize one ahash_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_hash_request(struct crypto_engine *engine,
				  struct ahash_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);

/**
 * crypto_finalize_skcipher_request - finalize one skcipher_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_skcipher_request(struct crypto_engine *engine,
				      struct skcipher_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);

/**
 * crypto_engine_start - start the hardware engine
 * @engine: the hardware engine need to be started
 *
 * Return 0 on success, else on fail.
 */
int crypto_engine_start(struct crypto_engine *engine)
{
	unsigned long flags;

	spin_lock_irqsave(&engine->queue_lock, flags);

	if (engine->running || engine->busy) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		return -EBUSY;
	}

	engine->running = true;
	spin_unlock_irqrestore(&engine->queue_lock, flags);

	kthread_queue_work(engine->kworker, &engine->pump_requests);

	return 0;
}
EXPORT_SYMBOL_GPL(crypto_engine_start);

/**
 * crypto_engine_stop - stop the hardware engine
 * @engine: the hardware engine need to be stopped
 *
 * Return 0 on success, else on fail.
 */
int crypto_engine_stop(struct crypto_engine *engine)
{
	unsigned long flags;
	unsigned int limit = 500;
	int ret = 0;

	spin_lock_irqsave(&engine->queue_lock, flags);

	/*
	 * If the engine queue is not empty or the engine is on busy state,
	 * we need to wait for a while to pump the requests of engine queue.
	 */
	while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		msleep(20);
		spin_lock_irqsave(&engine->queue_lock, flags);
	}

	if (crypto_queue_len(&engine->queue) || engine->busy)
		ret = -EBUSY;
	else
		engine->running = false;

	spin_unlock_irqrestore(&engine->queue_lock, flags);

	if (ret)
		dev_warn(engine->dev, "could not stop engine\n");

	return ret;
}
EXPORT_SYMBOL_GPL(crypto_engine_stop);

/**
 * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure
 * and initialize it by setting the maximum number of entries in the software
 * crypto-engine queue.
 * @dev: the device attached with one hardware engine
 * @retry_support: whether hardware has support for retry mechanism
 * @cbk_do_batch: pointer to a callback function to be invoked when executing a
 *                a batch of requests.
 *                This has the form:
 *                callback(struct crypto_engine *engine)
 *                where:
 *                @engine: the crypto engine structure.
 * @rt: whether this queue is set to run as a realtime task
 * @qlen: maximum size of the crypto-engine queue
 *
 * This must be called from context that can sleep.
 * Return: the crypto engine structure on success, else NULL.
 */
struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
						       bool retry_support,
						       int (*cbk_do_batch)(struct crypto_engine *engine),
						       bool rt, int qlen)
{
	struct sched_param param = { .sched_priority = MAX_RT_PRIO / 2 };
	struct crypto_engine *engine;

	if (!dev)
		return NULL;

	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
	if (!engine)
		return NULL;

	engine->dev = dev;
	engine->rt = rt;
	engine->running = false;
	engine->busy = false;
	engine->idling = false;
	engine->retry_support = retry_support;
	engine->priv_data = dev;
	/*
	 * Batch requests is possible only if
	 * hardware has support for retry mechanism.
	 */
	engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;

	snprintf(engine->name, sizeof(engine->name),
		 "%s-engine", dev_name(dev));

	crypto_init_queue(&engine->queue, qlen);
	spin_lock_init(&engine->queue_lock);

	engine->kworker = kthread_create_worker(0, "%s", engine->name);
	if (IS_ERR(engine->kworker)) {
		dev_err(dev, "failed to create crypto request pump task\n");
		return NULL;
	}
	kthread_init_work(&engine->pump_requests, crypto_pump_work);

	if (engine->rt) {
		dev_info(dev, "will run requests pump with realtime priority\n");
		sched_setscheduler(engine->kworker->task, SCHED_FIFO, &param);
	}

	return engine;
}
EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set);

/**
 * crypto_engine_alloc_init - allocate crypto hardware engine structure and
 * initialize it.
 * @dev: the device attached with one hardware engine
 * @rt: whether this queue is set to run as a realtime task
 *
 * This must be called from context that can sleep.
 * Return: the crypto engine structure on success, else NULL.
 */
struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
{
	return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,
						CRYPTO_ENGINE_MAX_QLEN);
}
EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);

/**
 * crypto_engine_exit - free the resources of hardware engine when exit
 * @engine: the hardware engine need to be freed
 *
 * Return 0 for success.
 */
int crypto_engine_exit(struct crypto_engine *engine)
{
	int ret;

	ret = crypto_engine_stop(engine);
	if (ret)
		return ret;

	kthread_destroy_worker(engine->kworker);

	return 0;
}
EXPORT_SYMBOL_GPL(crypto_engine_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Crypto hardware engine framework");
Commit	Line	Data
2874c5fd	1	// SPDX-License-Identifier: GPL-2.0-or-later
735d37b5 BW	2	/*
	3	* Handle async block request by crypto hardware engine.
	4	*
	5	* Copyright (C) 2016 Linaro, Inc.
	6	*
	7	* Author: Baolin Wang <baolin.wang@linaro.org>
735d37b5 BW	8	*/
	9
	10	#include <linux/err.h>
	11	#include <linux/delay.h>
2589ad84	12	#include <crypto/engine.h>
ae7e81c0	13	#include <uapi/linux/sched/types.h>
735d37b5 BW	14	#include "internal.h"
	15
	16	#define CRYPTO_ENGINE_MAX_QLEN 10
	17
218d1cc1 CL	18	/**
	19	* crypto_finalize_request - finalize one request if the request is done
	20	* @engine: the hardware engine
	21	* @req: the request need to be finalized
	22	* @err: error number
	23	*/
	24	static void crypto_finalize_request(struct crypto_engine *engine,
6a89f492	25	struct crypto_async_request *req, int err)
218d1cc1 CL	26	{
218d1cc1 CL	27	unsigned long flags;
6a89f492	28	bool finalize_req = false;
218d1cc1 CL	29	int ret;
	30	struct crypto_engine_ctx *enginectx;
	31
6a89f492 IP	32	/*
	33	* If hardware cannot enqueue more requests
	34	* and retry mechanism is not supported
	35	* make sure we are completing the current request
	36	*/
	37	if (!engine->retry_support) {
	38	spin_lock_irqsave(&engine->queue_lock, flags);
	39	if (engine->cur_req == req) {
	40	finalize_req = true;
	41	engine->cur_req = NULL;
	42	}
	43	spin_unlock_irqrestore(&engine->queue_lock, flags);
	44	}
218d1cc1	45
6a89f492	46	if (finalize_req \|\| engine->retry_support) {
218d1cc1	47	enginectx = crypto_tfm_ctx(req->tfm);
6a89f492	48	if (enginectx->op.prepare_request &&
218d1cc1 CL	49	enginectx->op.unprepare_request) {
	50	ret = enginectx->op.unprepare_request(engine, req);
	51	if (ret)
	52	dev_err(engine->dev, "failed to unprepare request\n");
	53	}
218d1cc1	54	}
218d1cc1 CL	55	req->complete(req, err);
	56
	57	kthread_queue_work(engine->kworker, &engine->pump_requests);
	58	}
	59
735d37b5 BW	60	/**
	61	* crypto_pump_requests - dequeue one request from engine queue to process
	62	* @engine: the hardware engine
	63	* @in_kthread: true if we are in the context of the request pump thread
	64	*
	65	* This function checks if there is any request in the engine queue that
	66	* needs processing and if so call out to the driver to initialize hardware
	67	* and handle each request.
	68	*/
	69	static void crypto_pump_requests(struct crypto_engine *engine,
	70	bool in_kthread)
	71	{
	72	struct crypto_async_request async_req, backlog;
735d37b5 BW	73	unsigned long flags;
735d37b5 BW	74	bool was_busy = false;
218d1cc1 CL	75	int ret;
218d1cc1 CL	76	struct crypto_engine_ctx *enginectx;
735d37b5 BW	77
	78	spin_lock_irqsave(&engine->queue_lock, flags);
	79
	80	/* Make sure we are not already running a request */
6a89f492	81	if (!engine->retry_support && engine->cur_req)
735d37b5 BW	82	goto out;
	83
	84	/* If another context is idling then defer */
	85	if (engine->idling) {
c4ca2b0b	86	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	87	goto out;
	88	}
	89
	90	/* Check if the engine queue is idle */
	91	if (!crypto_queue_len(&engine->queue) \|\| !engine->running) {
	92	if (!engine->busy)
	93	goto out;
	94
	95	/* Only do teardown in the thread */
	96	if (!in_kthread) {
c4ca2b0b	97	kthread_queue_work(engine->kworker,
735d37b5 BW	98	&engine->pump_requests);
	99	goto out;
	100	}
	101
	102	engine->busy = false;
	103	engine->idling = true;
	104	spin_unlock_irqrestore(&engine->queue_lock, flags);
	105
	106	if (engine->unprepare_crypt_hardware &&
	107	engine->unprepare_crypt_hardware(engine))
88d58ef8	108	dev_err(engine->dev, "failed to unprepare crypt hardware\n");
735d37b5 BW	109
	110	spin_lock_irqsave(&engine->queue_lock, flags);
	111	engine->idling = false;
	112	goto out;
	113	}
	114
6a89f492	115	start_request:
735d37b5 BW	116	/* Get the fist request from the engine queue to handle */
	117	backlog = crypto_get_backlog(&engine->queue);
	118	async_req = crypto_dequeue_request(&engine->queue);
	119	if (!async_req)
	120	goto out;
	121
6a89f492 IP	122	/*
	123	* If hardware doesn't support the retry mechanism,
	124	* keep track of the request we are processing now.
	125	* We'll need it on completion (crypto_finalize_request).
	126	*/
	127	if (!engine->retry_support)
	128	engine->cur_req = async_req;
	129
735d37b5 BW	130	if (backlog)
	131	backlog->complete(backlog, -EINPROGRESS);
	132
	133	if (engine->busy)
	134	was_busy = true;
	135	else
	136	engine->busy = true;
	137
	138	spin_unlock_irqrestore(&engine->queue_lock, flags);
	139
	140	/* Until here we get the request need to be encrypted successfully */
	141	if (!was_busy && engine->prepare_crypt_hardware) {
	142	ret = engine->prepare_crypt_hardware(engine);
	143	if (ret) {
88d58ef8	144	dev_err(engine->dev, "failed to prepare crypt hardware\n");
6a89f492	145	goto req_err_2;
735d37b5 BW	146	}
	147	}
	148
218d1cc1 CL	149	enginectx = crypto_tfm_ctx(async_req->tfm);
	150
	151	if (enginectx->op.prepare_request) {
	152	ret = enginectx->op.prepare_request(engine, async_req);
4cba7cf0	153	if (ret) {
218d1cc1 CL	154	dev_err(engine->dev, "failed to prepare request: %d\n",
218d1cc1 CL	155	ret);
6a89f492	156	goto req_err_2;
4cba7cf0	157	}
218d1cc1 CL	158	}
	159	if (!enginectx->op.do_one_request) {
	160	dev_err(engine->dev, "failed to do request\n");
	161	ret = -EINVAL;
6a89f492	162	goto req_err_1;
735d37b5	163	}
6a89f492	164
218d1cc1	165	ret = enginectx->op.do_one_request(engine, async_req);
6a89f492 IP	166
	167	/* Request unsuccessfully executed by hardware */
	168	if (ret < 0) {
	169	/*
	170	* If hardware queue is full (-ENOSPC), requeue request
	171	* regardless of backlog flag.
	172	* If hardware throws any other error code,
	173	* requeue only backlog requests.
	174	* Otherwise, unprepare and complete the request.
	175	*/
	176	if (!engine->retry_support \|\|
	177	((ret != -ENOSPC) &&
	178	!(async_req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) {
	179	dev_err(engine->dev,
	180	"Failed to do one request from queue: %d\n",
	181	ret);
	182	goto req_err_1;
	183	}
	184	/*
	185	* If retry mechanism is supported,
	186	* unprepare current request and
	187	* enqueue it back into crypto-engine queue.
	188	*/
	189	if (enginectx->op.unprepare_request) {
	190	ret = enginectx->op.unprepare_request(engine,
	191	async_req);
	192	if (ret)
	193	dev_err(engine->dev,
	194	"failed to unprepare request\n");
	195	}
	196	spin_lock_irqsave(&engine->queue_lock, flags);
	197	/*
	198	* If hardware was unable to execute request, enqueue it
	199	* back in front of crypto-engine queue, to keep the order
	200	* of requests.
	201	*/
	202	crypto_enqueue_request_head(&engine->queue, async_req);
	203
	204	kthread_queue_work(engine->kworker, &engine->pump_requests);
	205	goto out;
218d1cc1	206	}
735d37b5	207
6a89f492 IP	208	goto retry;
	209
	210	req_err_1:
	211	if (enginectx->op.unprepare_request) {
	212	ret = enginectx->op.unprepare_request(engine, async_req);
	213	if (ret)
	214	dev_err(engine->dev, "failed to unprepare request\n");
	215	}
	216
	217	req_err_2:
	218	async_req->complete(async_req, ret);
	219
	220	retry:
	221	/* If retry mechanism is supported, send new requests to engine */
	222	if (engine->retry_support) {
	223	spin_lock_irqsave(&engine->queue_lock, flags);
	224	goto start_request;
	225	}
735d37b5 BW	226	return;
	227
	228	out:
	229	spin_unlock_irqrestore(&engine->queue_lock, flags);
8d908226 IP	230
	231	/*
	232	* Batch requests is possible only if
	233	* hardware can enqueue multiple requests
	234	*/
	235	if (engine->do_batch_requests) {
	236	ret = engine->do_batch_requests(engine);
	237	if (ret)
	238	dev_err(engine->dev, "failed to do batch requests: %d\n",
	239	ret);
	240	}
	241
6a89f492	242	return;
735d37b5 BW	243	}
	244
	245	static void crypto_pump_work(struct kthread_work *work)
	246	{
	247	struct crypto_engine *engine =
	248	container_of(work, struct crypto_engine, pump_requests);
	249
	250	crypto_pump_requests(engine, true);
	251	}
	252
	253	/**
218d1cc1	254	* crypto_transfer_request - transfer the new request into the engine queue
735d37b5 BW	255	* @engine: the hardware engine
	256	* @req: the request need to be listed into the engine queue
	257	*/
218d1cc1 CL	258	static int crypto_transfer_request(struct crypto_engine *engine,
218d1cc1 CL	259	struct crypto_async_request *req,
4cba7cf0	260	bool need_pump)
735d37b5 BW	261	{
	262	unsigned long flags;
	263	int ret;
	264
	265	spin_lock_irqsave(&engine->queue_lock, flags);
	266
	267	if (!engine->running) {
	268	spin_unlock_irqrestore(&engine->queue_lock, flags);
	269	return -ESHUTDOWN;
	270	}
	271
218d1cc1	272	ret = crypto_enqueue_request(&engine->queue, req);
735d37b5 BW	273
735d37b5 BW	274	if (!engine->busy && need_pump)
c4ca2b0b	275	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	276
	277	spin_unlock_irqrestore(&engine->queue_lock, flags);
	278	return ret;
	279	}
4cba7cf0 CL	280
4cba7cf0 CL	281	/**
218d1cc1	282	* crypto_transfer_request_to_engine - transfer one request to list
4cba7cf0 CL	283	* into the engine queue
	284	* @engine: the hardware engine
	285	* @req: the request need to be listed into the engine queue
	286	*/
218d1cc1 CL	287	static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
218d1cc1 CL	288	struct crypto_async_request *req)
4cba7cf0	289	{
218d1cc1	290	return crypto_transfer_request(engine, req, true);
4cba7cf0	291	}
4cba7cf0	292
218d1cc1 CL	293	/**
	294	* crypto_transfer_aead_request_to_engine - transfer one aead_request
	295	* to list into the engine queue
	296	* @engine: the hardware engine
	297	* @req: the request need to be listed into the engine queue
	298	*/
	299	int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
	300	struct aead_request *req)
	301	{
	302	return crypto_transfer_request_to_engine(engine, &req->base);
	303	}
	304	EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);
4cba7cf0	305
218d1cc1 CL	306	/**
	307	* crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
	308	* to list into the engine queue
	309	* @engine: the hardware engine
	310	* @req: the request need to be listed into the engine queue
	311	*/
	312	int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
	313	struct akcipher_request *req)
	314	{
	315	return crypto_transfer_request_to_engine(engine, &req->base);
4cba7cf0	316	}
218d1cc1	317	EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);
735d37b5 BW	318
735d37b5 BW	319	/**
218d1cc1 CL	320	* crypto_transfer_hash_request_to_engine - transfer one ahash_request
218d1cc1 CL	321	* to list into the engine queue
735d37b5 BW	322	* @engine: the hardware engine
	323	* @req: the request need to be listed into the engine queue
	324	*/
4cba7cf0 CL	325	int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
4cba7cf0 CL	326	struct ahash_request *req)
735d37b5	327	{
218d1cc1	328	return crypto_transfer_request_to_engine(engine, &req->base);
735d37b5	329	}
4cba7cf0	330	EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);
735d37b5 BW	331
735d37b5 BW	332	/**
218d1cc1 CL	333	* crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
	334	* to list into the engine queue
	335	* @engine: the hardware engine
	336	* @req: the request need to be listed into the engine queue
	337	*/
	338	int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
	339	struct skcipher_request *req)
	340	{
	341	return crypto_transfer_request_to_engine(engine, &req->base);
	342	}
	343	EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);
	344
218d1cc1 CL	345	/**
	346	* crypto_finalize_aead_request - finalize one aead_request if
	347	* the request is done
	348	* @engine: the hardware engine
	349	* @req: the request need to be finalized
	350	* @err: error number
	351	*/
	352	void crypto_finalize_aead_request(struct crypto_engine *engine,
	353	struct aead_request *req, int err)
	354	{
	355	return crypto_finalize_request(engine, &req->base, err);
	356	}
	357	EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);
735d37b5	358
218d1cc1 CL	359	/**
	360	* crypto_finalize_akcipher_request - finalize one akcipher_request if
	361	* the request is done
	362	* @engine: the hardware engine
	363	* @req: the request need to be finalized
	364	* @err: error number
	365	*/
	366	void crypto_finalize_akcipher_request(struct crypto_engine *engine,
	367	struct akcipher_request *req, int err)
	368	{
	369	return crypto_finalize_request(engine, &req->base, err);
4cba7cf0	370	}
218d1cc1	371	EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);
4cba7cf0 CL	372
4cba7cf0 CL	373	/**
218d1cc1 CL	374	* crypto_finalize_hash_request - finalize one ahash_request if
218d1cc1 CL	375	* the request is done
4cba7cf0 CL	376	* @engine: the hardware engine
	377	* @req: the request need to be finalized
	378	* @err: error number
	379	*/
	380	void crypto_finalize_hash_request(struct crypto_engine *engine,
	381	struct ahash_request *req, int err)
	382	{
218d1cc1	383	return crypto_finalize_request(engine, &req->base, err);
735d37b5	384	}
4cba7cf0	385	EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
735d37b5	386
218d1cc1 CL	387	/**
	388	* crypto_finalize_skcipher_request - finalize one skcipher_request if
	389	* the request is done
	390	* @engine: the hardware engine
	391	* @req: the request need to be finalized
	392	* @err: error number
	393	*/
	394	void crypto_finalize_skcipher_request(struct crypto_engine *engine,
	395	struct skcipher_request *req, int err)
	396	{
	397	return crypto_finalize_request(engine, &req->base, err);
	398	}
	399	EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);
	400
735d37b5 BW	401	/**
	402	* crypto_engine_start - start the hardware engine
	403	* @engine: the hardware engine need to be started
	404	*
	405	* Return 0 on success, else on fail.
	406	*/
	407	int crypto_engine_start(struct crypto_engine *engine)
	408	{
	409	unsigned long flags;
	410
	411	spin_lock_irqsave(&engine->queue_lock, flags);
	412
	413	if (engine->running \|\| engine->busy) {
	414	spin_unlock_irqrestore(&engine->queue_lock, flags);
	415	return -EBUSY;
	416	}
	417
	418	engine->running = true;
	419	spin_unlock_irqrestore(&engine->queue_lock, flags);
	420
c4ca2b0b	421	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	422
	423	return 0;
	424	}
	425	EXPORT_SYMBOL_GPL(crypto_engine_start);
	426
	427	/**
	428	* crypto_engine_stop - stop the hardware engine
	429	* @engine: the hardware engine need to be stopped
	430	*
	431	* Return 0 on success, else on fail.
	432	*/
	433	int crypto_engine_stop(struct crypto_engine *engine)
	434	{
	435	unsigned long flags;
4cba7cf0	436	unsigned int limit = 500;
735d37b5 BW	437	int ret = 0;
	438
	439	spin_lock_irqsave(&engine->queue_lock, flags);
	440
	441	/*
	442	* If the engine queue is not empty or the engine is on busy state,
	443	* we need to wait for a while to pump the requests of engine queue.
	444	*/
	445	while ((crypto_queue_len(&engine->queue) \|\| engine->busy) && limit--) {
	446	spin_unlock_irqrestore(&engine->queue_lock, flags);
	447	msleep(20);
	448	spin_lock_irqsave(&engine->queue_lock, flags);
	449	}
	450
	451	if (crypto_queue_len(&engine->queue) \|\| engine->busy)
	452	ret = -EBUSY;
	453	else
	454	engine->running = false;
	455
	456	spin_unlock_irqrestore(&engine->queue_lock, flags);
	457
	458	if (ret)
88d58ef8	459	dev_warn(engine->dev, "could not stop engine\n");
735d37b5 BW	460
	461	return ret;
	462	}
	463	EXPORT_SYMBOL_GPL(crypto_engine_stop);
	464
	465	/**
6a89f492 IP	466	* crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure
	467	* and initialize it by setting the maximum number of entries in the software
	468	* crypto-engine queue.
735d37b5	469	* @dev: the device attached with one hardware engine
6a89f492	470	* @retry_support: whether hardware has support for retry mechanism
8d908226 IP	471	* @cbk_do_batch: pointer to a callback function to be invoked when executing a
	472	* a batch of requests.
	473	* This has the form:
	474	* callback(struct crypto_engine *engine)
	475	* where:
	476	* @engine: the crypto engine structure.
735d37b5	477	* @rt: whether this queue is set to run as a realtime task
6a89f492	478	* @qlen: maximum size of the crypto-engine queue
735d37b5 BW	479	*
	480	* This must be called from context that can sleep.
	481	* Return: the crypto engine structure on success, else NULL.
	482	*/
6a89f492 IP	483	struct crypto_engine crypto_engine_alloc_init_and_set(struct device dev,
6a89f492 IP	484	bool retry_support,
8d908226	485	int (cbk_do_batch)(struct crypto_engine engine),
6a89f492	486	bool rt, int qlen)
735d37b5	487	{
d13dfae3	488	struct sched_param param = { .sched_priority = MAX_RT_PRIO / 2 };
735d37b5 BW	489	struct crypto_engine *engine;
	490
	491	if (!dev)
	492	return NULL;
	493
	494	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
	495	if (!engine)
	496	return NULL;
	497
88d58ef8	498	engine->dev = dev;
735d37b5 BW	499	engine->rt = rt;
	500	engine->running = false;
	501	engine->busy = false;
	502	engine->idling = false;
6a89f492	503	engine->retry_support = retry_support;
735d37b5	504	engine->priv_data = dev;
8d908226 IP	505	/*
	506	* Batch requests is possible only if
	507	* hardware has support for retry mechanism.
	508	*/
	509	engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;
	510
735d37b5 BW	511	snprintf(engine->name, sizeof(engine->name),
	512	"%s-engine", dev_name(dev));
	513
6a89f492	514	crypto_init_queue(&engine->queue, qlen);
735d37b5 BW	515	spin_lock_init(&engine->queue_lock);
735d37b5 BW	516
c4ca2b0b PM	517	engine->kworker = kthread_create_worker(0, "%s", engine->name);
c4ca2b0b PM	518	if (IS_ERR(engine->kworker)) {
735d37b5 BW	519	dev_err(dev, "failed to create crypto request pump task\n");
	520	return NULL;
	521	}
3989144f	522	kthread_init_work(&engine->pump_requests, crypto_pump_work);
735d37b5 BW	523
	524	if (engine->rt) {
	525	dev_info(dev, "will run requests pump with realtime priority\n");
c4ca2b0b	526	sched_setscheduler(engine->kworker->task, SCHED_FIFO, &param);
735d37b5 BW	527	}
	528
	529	return engine;
	530	}
6a89f492 IP	531	EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set);
	532
	533	/**
	534	* crypto_engine_alloc_init - allocate crypto hardware engine structure and
	535	* initialize it.
	536	* @dev: the device attached with one hardware engine
	537	* @rt: whether this queue is set to run as a realtime task
	538	*
	539	* This must be called from context that can sleep.
	540	* Return: the crypto engine structure on success, else NULL.
	541	*/
	542	struct crypto_engine crypto_engine_alloc_init(struct device dev, bool rt)
	543	{
8d908226	544	return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,
6a89f492 IP	545	CRYPTO_ENGINE_MAX_QLEN);
6a89f492 IP	546	}
735d37b5 BW	547	EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
	548
	549	/**
	550	* crypto_engine_exit - free the resources of hardware engine when exit
	551	* @engine: the hardware engine need to be freed
	552	*
	553	* Return 0 for success.
	554	*/
	555	int crypto_engine_exit(struct crypto_engine *engine)
	556	{
	557	int ret;
	558
	559	ret = crypto_engine_stop(engine);
	560	if (ret)
	561	return ret;
	562
c4ca2b0b	563	kthread_destroy_worker(engine->kworker);
735d37b5 BW	564
	565	return 0;
	566	}
	567	EXPORT_SYMBOL_GPL(crypto_engine_exit);
	568
	569	MODULE_LICENSE("GPL");
	570	MODULE_DESCRIPTION("Crypto hardware engine framework");