[linux-block.git] / crypto / crypto_engine.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Handle async block request by crypto hardware engine.
 *
 * Copyright (C) 2016 Linaro, Inc.
 *
 * Author: Baolin Wang <baolin.wang@linaro.org>
 */

#include <linux/err.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <crypto/engine.h>
#include <uapi/linux/sched/types.h>
#include "internal.h"

#define CRYPTO_ENGINE_MAX_QLEN 10

/**
 * crypto_finalize_request - finalize one request if the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
static void crypto_finalize_request(struct crypto_engine *engine,
				    struct crypto_async_request *req, int err)
{
	unsigned long flags;
	bool finalize_req = false;
	int ret;
	struct crypto_engine_ctx *enginectx;

	/*
	 * If hardware cannot enqueue more requests
	 * and retry mechanism is not supported
	 * make sure we are completing the current request
	 */
	if (!engine->retry_support) {
		spin_lock_irqsave(&engine->queue_lock, flags);
		if (engine->cur_req == req) {
			finalize_req = true;
			engine->cur_req = NULL;
		}
		spin_unlock_irqrestore(&engine->queue_lock, flags);
	}

	if (finalize_req || engine->retry_support) {
		enginectx = crypto_tfm_ctx(req->tfm);
		if (enginectx->op.prepare_request &&
		    enginectx->op.unprepare_request) {
			ret = enginectx->op.unprepare_request(engine, req);
			if (ret)
				dev_err(engine->dev, "failed to unprepare request\n");
		}
	}
	lockdep_assert_in_softirq();
	crypto_request_complete(req, err);

	kthread_queue_work(engine->kworker, &engine->pump_requests);
}

/**
 * crypto_pump_requests - dequeue one request from engine queue to process
 * @engine: the hardware engine
 * @in_kthread: true if we are in the context of the request pump thread
 *
 * This function checks if there is any request in the engine queue that
 * needs processing and if so call out to the driver to initialize hardware
 * and handle each request.
 */
static void crypto_pump_requests(struct crypto_engine *engine,
				 bool in_kthread)
{
	struct crypto_async_request *async_req, *backlog;
	unsigned long flags;
	bool was_busy = false;
	int ret;
	struct crypto_engine_ctx *enginectx;

	spin_lock_irqsave(&engine->queue_lock, flags);

	/* Make sure we are not already running a request */
	if (!engine->retry_support && engine->cur_req)
		goto out;

	/* If another context is idling then defer */
	if (engine->idling) {
		kthread_queue_work(engine->kworker, &engine->pump_requests);
		goto out;
	}

	/* Check if the engine queue is idle */
	if (!crypto_queue_len(&engine->queue) || !engine->running) {
		if (!engine->busy)
			goto out;

		/* Only do teardown in the thread */
		if (!in_kthread) {
			kthread_queue_work(engine->kworker,
					   &engine->pump_requests);
			goto out;
		}

		engine->busy = false;
		engine->idling = true;
		spin_unlock_irqrestore(&engine->queue_lock, flags);

		if (engine->unprepare_crypt_hardware &&
		    engine->unprepare_crypt_hardware(engine))
			dev_err(engine->dev, "failed to unprepare crypt hardware\n");

		spin_lock_irqsave(&engine->queue_lock, flags);
		engine->idling = false;
		goto out;
	}

start_request:
	/* Get the fist request from the engine queue to handle */
	backlog = crypto_get_backlog(&engine->queue);
	async_req = crypto_dequeue_request(&engine->queue);
	if (!async_req)
		goto out;

	/*
	 * If hardware doesn't support the retry mechanism,
	 * keep track of the request we are processing now.
	 * We'll need it on completion (crypto_finalize_request).
	 */
	if (!engine->retry_support)
		engine->cur_req = async_req;

	if (engine->busy)
		was_busy = true;
	else
		engine->busy = true;

	spin_unlock_irqrestore(&engine->queue_lock, flags);

	/* Until here we get the request need to be encrypted successfully */
	if (!was_busy && engine->prepare_crypt_hardware) {
		ret = engine->prepare_crypt_hardware(engine);
		if (ret) {
			dev_err(engine->dev, "failed to prepare crypt hardware\n");
			goto req_err_2;
		}
	}

	enginectx = crypto_tfm_ctx(async_req->tfm);

	if (enginectx->op.prepare_request) {
		ret = enginectx->op.prepare_request(engine, async_req);
		if (ret) {
			dev_err(engine->dev, "failed to prepare request: %d\n",
				ret);
			goto req_err_2;
		}
	}
	if (!enginectx->op.do_one_request) {
		dev_err(engine->dev, "failed to do request\n");
		ret = -EINVAL;
		goto req_err_1;
	}

	ret = enginectx->op.do_one_request(engine, async_req);

	/* Request unsuccessfully executed by hardware */
	if (ret < 0) {
		/*
		 * If hardware queue is full (-ENOSPC), requeue request
		 * regardless of backlog flag.
		 * Otherwise, unprepare and complete the request.
		 */
		if (!engine->retry_support ||
		    (ret != -ENOSPC)) {
			dev_err(engine->dev,
				"Failed to do one request from queue: %d\n",
				ret);
			goto req_err_1;
		}
		/*
		 * If retry mechanism is supported,
		 * unprepare current request and
		 * enqueue it back into crypto-engine queue.
		 */
		if (enginectx->op.unprepare_request) {
			ret = enginectx->op.unprepare_request(engine,
							      async_req);
			if (ret)
				dev_err(engine->dev,
					"failed to unprepare request\n");
		}
		spin_lock_irqsave(&engine->queue_lock, flags);
		/*
		 * If hardware was unable to execute request, enqueue it
		 * back in front of crypto-engine queue, to keep the order
		 * of requests.
		 */
		crypto_enqueue_request_head(&engine->queue, async_req);

		kthread_queue_work(engine->kworker, &engine->pump_requests);
		goto out;
	}

	goto retry;

req_err_1:
	if (enginectx->op.unprepare_request) {
		ret = enginectx->op.unprepare_request(engine, async_req);
		if (ret)
			dev_err(engine->dev, "failed to unprepare request\n");
	}

req_err_2:
	crypto_request_complete(async_req, ret);

retry:
	if (backlog)
		crypto_request_complete(backlog, -EINPROGRESS);

	/* If retry mechanism is supported, send new requests to engine */
	if (engine->retry_support) {
		spin_lock_irqsave(&engine->queue_lock, flags);
		goto start_request;
	}
	return;

out:
	spin_unlock_irqrestore(&engine->queue_lock, flags);

	/*
	 * Batch requests is possible only if
	 * hardware can enqueue multiple requests
	 */
	if (engine->do_batch_requests) {
		ret = engine->do_batch_requests(engine);
		if (ret)
			dev_err(engine->dev, "failed to do batch requests: %d\n",
				ret);
	}

	return;
}

static void crypto_pump_work(struct kthread_work *work)
{
	struct crypto_engine *engine =
		container_of(work, struct crypto_engine, pump_requests);

	crypto_pump_requests(engine, true);
}

/**
 * crypto_transfer_request - transfer the new request into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 * @need_pump: indicates whether queue the pump of request to kthread_work
 */
static int crypto_transfer_request(struct crypto_engine *engine,
				   struct crypto_async_request *req,
				   bool need_pump)
{
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&engine->queue_lock, flags);

	if (!engine->running) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		return -ESHUTDOWN;
	}

	ret = crypto_enqueue_request(&engine->queue, req);

	if (!engine->busy && need_pump)
		kthread_queue_work(engine->kworker, &engine->pump_requests);

	spin_unlock_irqrestore(&engine->queue_lock, flags);
	return ret;
}

/**
 * crypto_transfer_request_to_engine - transfer one request to list
 * into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
					     struct crypto_async_request *req)
{
	return crypto_transfer_request(engine, req, true);
}

/**
 * crypto_transfer_aead_request_to_engine - transfer one aead_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
					   struct aead_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);

/**
 * crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
					       struct akcipher_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);

/**
 * crypto_transfer_hash_request_to_engine - transfer one ahash_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
					   struct ahash_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);

/**
 * crypto_transfer_kpp_request_to_engine - transfer one kpp_request to list
 * into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine,
					  struct kpp_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_kpp_request_to_engine);

/**
 * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
					       struct skcipher_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);

/**
 * crypto_finalize_aead_request - finalize one aead_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_aead_request(struct crypto_engine *engine,
				  struct aead_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);

/**
 * crypto_finalize_akcipher_request - finalize one akcipher_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_akcipher_request(struct crypto_engine *engine,
				      struct akcipher_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);

/**
 * crypto_finalize_hash_request - finalize one ahash_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_hash_request(struct crypto_engine *engine,
				  struct ahash_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);

/**
 * crypto_finalize_kpp_request - finalize one kpp_request if the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_kpp_request(struct crypto_engine *engine,
				 struct kpp_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_kpp_request);

/**
 * crypto_finalize_skcipher_request - finalize one skcipher_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_skcipher_request(struct crypto_engine *engine,
				      struct skcipher_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);

/**
 * crypto_engine_start - start the hardware engine
 * @engine: the hardware engine need to be started
 *
 * Return 0 on success, else on fail.
 */
int crypto_engine_start(struct crypto_engine *engine)
{
	unsigned long flags;

	spin_lock_irqsave(&engine->queue_lock, flags);

	if (engine->running || engine->busy) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		return -EBUSY;
	}

	engine->running = true;
	spin_unlock_irqrestore(&engine->queue_lock, flags);

	kthread_queue_work(engine->kworker, &engine->pump_requests);

	return 0;
}
EXPORT_SYMBOL_GPL(crypto_engine_start);

/**
 * crypto_engine_stop - stop the hardware engine
 * @engine: the hardware engine need to be stopped
 *
 * Return 0 on success, else on fail.
 */
int crypto_engine_stop(struct crypto_engine *engine)
{
	unsigned long flags;
	unsigned int limit = 500;
	int ret = 0;

	spin_lock_irqsave(&engine->queue_lock, flags);

	/*
	 * If the engine queue is not empty or the engine is on busy state,
	 * we need to wait for a while to pump the requests of engine queue.
	 */
	while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		msleep(20);
		spin_lock_irqsave(&engine->queue_lock, flags);
	}

	if (crypto_queue_len(&engine->queue) || engine->busy)
		ret = -EBUSY;
	else
		engine->running = false;

	spin_unlock_irqrestore(&engine->queue_lock, flags);

	if (ret)
		dev_warn(engine->dev, "could not stop engine\n");

	return ret;
}
EXPORT_SYMBOL_GPL(crypto_engine_stop);

/**
 * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure
 * and initialize it by setting the maximum number of entries in the software
 * crypto-engine queue.
 * @dev: the device attached with one hardware engine
 * @retry_support: whether hardware has support for retry mechanism
 * @cbk_do_batch: pointer to a callback function to be invoked when executing
 *                a batch of requests.
 *                This has the form:
 *                callback(struct crypto_engine *engine)
 *                where:
 *                engine: the crypto engine structure.
 * @rt: whether this queue is set to run as a realtime task
 * @qlen: maximum size of the crypto-engine queue
 *
 * This must be called from context that can sleep.
 * Return: the crypto engine structure on success, else NULL.
 */
struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
						       bool retry_support,
						       int (*cbk_do_batch)(struct crypto_engine *engine),
						       bool rt, int qlen)
{
	struct crypto_engine *engine;

	if (!dev)
		return NULL;

	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
	if (!engine)
		return NULL;

	engine->dev = dev;
	engine->rt = rt;
	engine->running = false;
	engine->busy = false;
	engine->idling = false;
	engine->retry_support = retry_support;
	engine->priv_data = dev;
	/*
	 * Batch requests is possible only if
	 * hardware has support for retry mechanism.
	 */
	engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;

	snprintf(engine->name, sizeof(engine->name),
		 "%s-engine", dev_name(dev));

	crypto_init_queue(&engine->queue, qlen);
	spin_lock_init(&engine->queue_lock);

	engine->kworker = kthread_create_worker(0, "%s", engine->name);
	if (IS_ERR(engine->kworker)) {
		dev_err(dev, "failed to create crypto request pump task\n");
		return NULL;
	}
	kthread_init_work(&engine->pump_requests, crypto_pump_work);

	if (engine->rt) {
		dev_info(dev, "will run requests pump with realtime priority\n");
		sched_set_fifo(engine->kworker->task);
	}

	return engine;
}
EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set);

/**
 * crypto_engine_alloc_init - allocate crypto hardware engine structure and
 * initialize it.
 * @dev: the device attached with one hardware engine
 * @rt: whether this queue is set to run as a realtime task
 *
 * This must be called from context that can sleep.
 * Return: the crypto engine structure on success, else NULL.
 */
struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
{
	return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,
						CRYPTO_ENGINE_MAX_QLEN);
}
EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);

/**
 * crypto_engine_exit - free the resources of hardware engine when exit
 * @engine: the hardware engine need to be freed
 *
 * Return 0 for success.
 */
int crypto_engine_exit(struct crypto_engine *engine)
{
	int ret;

	ret = crypto_engine_stop(engine);
	if (ret)
		return ret;

	kthread_destroy_worker(engine->kworker);

	return 0;
}
EXPORT_SYMBOL_GPL(crypto_engine_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Crypto hardware engine framework");
Commit	Line	Data
2874c5fd	1	// SPDX-License-Identifier: GPL-2.0-or-later
735d37b5 BW	2	/*
	3	* Handle async block request by crypto hardware engine.
	4	*
	5	* Copyright (C) 2016 Linaro, Inc.
	6	*
	7	* Author: Baolin Wang <baolin.wang@linaro.org>
735d37b5 BW	8	*/
	9
	10	#include <linux/err.h>
	11	#include <linux/delay.h>
0c3dc787	12	#include <linux/device.h>
2589ad84	13	#include <crypto/engine.h>
ae7e81c0	14	#include <uapi/linux/sched/types.h>
735d37b5 BW	15	#include "internal.h"
	16
	17	#define CRYPTO_ENGINE_MAX_QLEN 10
	18
218d1cc1 CL	19	/**
	20	* crypto_finalize_request - finalize one request if the request is done
	21	* @engine: the hardware engine
	22	* @req: the request need to be finalized
	23	* @err: error number
	24	*/
	25	static void crypto_finalize_request(struct crypto_engine *engine,
6a89f492	26	struct crypto_async_request *req, int err)
218d1cc1 CL	27	{
218d1cc1 CL	28	unsigned long flags;
6a89f492	29	bool finalize_req = false;
218d1cc1 CL	30	int ret;
	31	struct crypto_engine_ctx *enginectx;
	32
6a89f492 IP	33	/*
	34	* If hardware cannot enqueue more requests
	35	* and retry mechanism is not supported
	36	* make sure we are completing the current request
	37	*/
	38	if (!engine->retry_support) {
	39	spin_lock_irqsave(&engine->queue_lock, flags);
	40	if (engine->cur_req == req) {
	41	finalize_req = true;
	42	engine->cur_req = NULL;
	43	}
	44	spin_unlock_irqrestore(&engine->queue_lock, flags);
	45	}
218d1cc1	46
6a89f492	47	if (finalize_req \|\| engine->retry_support) {
218d1cc1	48	enginectx = crypto_tfm_ctx(req->tfm);
6a89f492	49	if (enginectx->op.prepare_request &&
218d1cc1 CL	50	enginectx->op.unprepare_request) {
	51	ret = enginectx->op.unprepare_request(engine, req);
	52	if (ret)
	53	dev_err(engine->dev, "failed to unprepare request\n");
	54	}
218d1cc1	55	}
4058cf08	56	lockdep_assert_in_softirq();
6909823d	57	crypto_request_complete(req, err);
218d1cc1 CL	58
	59	kthread_queue_work(engine->kworker, &engine->pump_requests);
	60	}
	61
735d37b5 BW	62	/**
	63	* crypto_pump_requests - dequeue one request from engine queue to process
	64	* @engine: the hardware engine
	65	* @in_kthread: true if we are in the context of the request pump thread
	66	*
	67	* This function checks if there is any request in the engine queue that
	68	* needs processing and if so call out to the driver to initialize hardware
	69	* and handle each request.
	70	*/
	71	static void crypto_pump_requests(struct crypto_engine *engine,
	72	bool in_kthread)
	73	{
	74	struct crypto_async_request async_req, backlog;
735d37b5 BW	75	unsigned long flags;
735d37b5 BW	76	bool was_busy = false;
218d1cc1 CL	77	int ret;
218d1cc1 CL	78	struct crypto_engine_ctx *enginectx;
735d37b5 BW	79
	80	spin_lock_irqsave(&engine->queue_lock, flags);
	81
	82	/* Make sure we are not already running a request */
6a89f492	83	if (!engine->retry_support && engine->cur_req)
735d37b5 BW	84	goto out;
	85
	86	/* If another context is idling then defer */
	87	if (engine->idling) {
c4ca2b0b	88	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	89	goto out;
	90	}
	91
	92	/* Check if the engine queue is idle */
	93	if (!crypto_queue_len(&engine->queue) \|\| !engine->running) {
	94	if (!engine->busy)
	95	goto out;
	96
	97	/* Only do teardown in the thread */
	98	if (!in_kthread) {
c4ca2b0b	99	kthread_queue_work(engine->kworker,
735d37b5 BW	100	&engine->pump_requests);
	101	goto out;
	102	}
	103
	104	engine->busy = false;
	105	engine->idling = true;
	106	spin_unlock_irqrestore(&engine->queue_lock, flags);
	107
	108	if (engine->unprepare_crypt_hardware &&
	109	engine->unprepare_crypt_hardware(engine))
88d58ef8	110	dev_err(engine->dev, "failed to unprepare crypt hardware\n");
735d37b5 BW	111
	112	spin_lock_irqsave(&engine->queue_lock, flags);
	113	engine->idling = false;
	114	goto out;
	115	}
	116
6a89f492	117	start_request:
735d37b5 BW	118	/* Get the fist request from the engine queue to handle */
	119	backlog = crypto_get_backlog(&engine->queue);
	120	async_req = crypto_dequeue_request(&engine->queue);
	121	if (!async_req)
	122	goto out;
	123
6a89f492 IP	124	/*
	125	* If hardware doesn't support the retry mechanism,
	126	* keep track of the request we are processing now.
	127	* We'll need it on completion (crypto_finalize_request).
	128	*/
	129	if (!engine->retry_support)
	130	engine->cur_req = async_req;
	131
735d37b5 BW	132	if (engine->busy)
	133	was_busy = true;
	134	else
	135	engine->busy = true;
	136
	137	spin_unlock_irqrestore(&engine->queue_lock, flags);
	138
	139	/* Until here we get the request need to be encrypted successfully */
	140	if (!was_busy && engine->prepare_crypt_hardware) {
	141	ret = engine->prepare_crypt_hardware(engine);
	142	if (ret) {
88d58ef8	143	dev_err(engine->dev, "failed to prepare crypt hardware\n");
6a89f492	144	goto req_err_2;
735d37b5 BW	145	}
	146	}
	147
218d1cc1 CL	148	enginectx = crypto_tfm_ctx(async_req->tfm);
	149
	150	if (enginectx->op.prepare_request) {
	151	ret = enginectx->op.prepare_request(engine, async_req);
4cba7cf0	152	if (ret) {
218d1cc1 CL	153	dev_err(engine->dev, "failed to prepare request: %d\n",
218d1cc1 CL	154	ret);
6a89f492	155	goto req_err_2;
4cba7cf0	156	}
218d1cc1 CL	157	}
	158	if (!enginectx->op.do_one_request) {
	159	dev_err(engine->dev, "failed to do request\n");
	160	ret = -EINVAL;
6a89f492	161	goto req_err_1;
735d37b5	162	}
6a89f492	163
218d1cc1	164	ret = enginectx->op.do_one_request(engine, async_req);
6a89f492 IP	165
	166	/* Request unsuccessfully executed by hardware */
	167	if (ret < 0) {
	168	/*
	169	* If hardware queue is full (-ENOSPC), requeue request
	170	* regardless of backlog flag.
6a89f492 IP	171	* Otherwise, unprepare and complete the request.
	172	*/
	173	if (!engine->retry_support \|\|
d1c72f6e	174	(ret != -ENOSPC)) {
6a89f492 IP	175	dev_err(engine->dev,
	176	"Failed to do one request from queue: %d\n",
	177	ret);
	178	goto req_err_1;
	179	}
	180	/*
	181	* If retry mechanism is supported,
	182	* unprepare current request and
	183	* enqueue it back into crypto-engine queue.
	184	*/
	185	if (enginectx->op.unprepare_request) {
	186	ret = enginectx->op.unprepare_request(engine,
	187	async_req);
	188	if (ret)
	189	dev_err(engine->dev,
	190	"failed to unprepare request\n");
	191	}
	192	spin_lock_irqsave(&engine->queue_lock, flags);
	193	/*
	194	* If hardware was unable to execute request, enqueue it
	195	* back in front of crypto-engine queue, to keep the order
	196	* of requests.
	197	*/
	198	crypto_enqueue_request_head(&engine->queue, async_req);
	199
	200	kthread_queue_work(engine->kworker, &engine->pump_requests);
	201	goto out;
218d1cc1	202	}
735d37b5	203
6a89f492 IP	204	goto retry;
	205
	206	req_err_1:
	207	if (enginectx->op.unprepare_request) {
	208	ret = enginectx->op.unprepare_request(engine, async_req);
	209	if (ret)
	210	dev_err(engine->dev, "failed to unprepare request\n");
	211	}
	212
	213	req_err_2:
6909823d	214	crypto_request_complete(async_req, ret);
6a89f492 IP	215
6a89f492 IP	216	retry:
4140aafc OB	217	if (backlog)
	218	crypto_request_complete(backlog, -EINPROGRESS);
	219
6a89f492 IP	220	/* If retry mechanism is supported, send new requests to engine */
	221	if (engine->retry_support) {
	222	spin_lock_irqsave(&engine->queue_lock, flags);
	223	goto start_request;
	224	}
735d37b5 BW	225	return;
	226
	227	out:
	228	spin_unlock_irqrestore(&engine->queue_lock, flags);
8d908226 IP	229
	230	/*
	231	* Batch requests is possible only if
	232	* hardware can enqueue multiple requests
	233	*/
	234	if (engine->do_batch_requests) {
	235	ret = engine->do_batch_requests(engine);
	236	if (ret)
	237	dev_err(engine->dev, "failed to do batch requests: %d\n",
	238	ret);
	239	}
	240
6a89f492	241	return;
735d37b5 BW	242	}
	243
	244	static void crypto_pump_work(struct kthread_work *work)
	245	{
	246	struct crypto_engine *engine =
	247	container_of(work, struct crypto_engine, pump_requests);
	248
	249	crypto_pump_requests(engine, true);
	250	}
	251
	252	/**
218d1cc1	253	* crypto_transfer_request - transfer the new request into the engine queue
735d37b5 BW	254	* @engine: the hardware engine
735d37b5 BW	255	* @req: the request need to be listed into the engine queue
d5db91d2	256	* @need_pump: indicates whether queue the pump of request to kthread_work
735d37b5	257	*/
218d1cc1 CL	258	static int crypto_transfer_request(struct crypto_engine *engine,
218d1cc1 CL	259	struct crypto_async_request *req,
4cba7cf0	260	bool need_pump)
735d37b5 BW	261	{
	262	unsigned long flags;
	263	int ret;
	264
	265	spin_lock_irqsave(&engine->queue_lock, flags);
	266
	267	if (!engine->running) {
	268	spin_unlock_irqrestore(&engine->queue_lock, flags);
	269	return -ESHUTDOWN;
	270	}
	271
218d1cc1	272	ret = crypto_enqueue_request(&engine->queue, req);
735d37b5 BW	273
735d37b5 BW	274	if (!engine->busy && need_pump)
c4ca2b0b	275	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	276
	277	spin_unlock_irqrestore(&engine->queue_lock, flags);
	278	return ret;
	279	}
4cba7cf0 CL	280
4cba7cf0 CL	281	/**
218d1cc1	282	* crypto_transfer_request_to_engine - transfer one request to list
4cba7cf0 CL	283	* into the engine queue
	284	* @engine: the hardware engine
	285	* @req: the request need to be listed into the engine queue
	286	*/
218d1cc1 CL	287	static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
218d1cc1 CL	288	struct crypto_async_request *req)
4cba7cf0	289	{
218d1cc1	290	return crypto_transfer_request(engine, req, true);
4cba7cf0	291	}
4cba7cf0	292
218d1cc1 CL	293	/**
	294	* crypto_transfer_aead_request_to_engine - transfer one aead_request
	295	* to list into the engine queue
	296	* @engine: the hardware engine
	297	* @req: the request need to be listed into the engine queue
	298	*/
	299	int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
	300	struct aead_request *req)
	301	{
	302	return crypto_transfer_request_to_engine(engine, &req->base);
	303	}
	304	EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);
4cba7cf0	305
218d1cc1 CL	306	/**
	307	* crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
	308	* to list into the engine queue
	309	* @engine: the hardware engine
	310	* @req: the request need to be listed into the engine queue
	311	*/
	312	int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
	313	struct akcipher_request *req)
	314	{
	315	return crypto_transfer_request_to_engine(engine, &req->base);
4cba7cf0	316	}
218d1cc1	317	EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);
735d37b5 BW	318
735d37b5 BW	319	/**
218d1cc1 CL	320	* crypto_transfer_hash_request_to_engine - transfer one ahash_request
218d1cc1 CL	321	* to list into the engine queue
735d37b5 BW	322	* @engine: the hardware engine
	323	* @req: the request need to be listed into the engine queue
	324	*/
4cba7cf0 CL	325	int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
4cba7cf0 CL	326	struct ahash_request *req)
735d37b5	327	{
218d1cc1	328	return crypto_transfer_request_to_engine(engine, &req->base);
735d37b5	329	}
4cba7cf0	330	EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);
735d37b5	331
1730c5aa PK	332	/**
	333	* crypto_transfer_kpp_request_to_engine - transfer one kpp_request to list
	334	* into the engine queue
	335	* @engine: the hardware engine
	336	* @req: the request need to be listed into the engine queue
	337	*/
	338	int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine,
	339	struct kpp_request *req)
	340	{
	341	return crypto_transfer_request_to_engine(engine, &req->base);
	342	}
	343	EXPORT_SYMBOL_GPL(crypto_transfer_kpp_request_to_engine);
	344
735d37b5	345	/**
218d1cc1 CL	346	* crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
	347	* to list into the engine queue
	348	* @engine: the hardware engine
	349	* @req: the request need to be listed into the engine queue
	350	*/
	351	int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
	352	struct skcipher_request *req)
	353	{
	354	return crypto_transfer_request_to_engine(engine, &req->base);
	355	}
	356	EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);
	357
218d1cc1 CL	358	/**
	359	* crypto_finalize_aead_request - finalize one aead_request if
	360	* the request is done
	361	* @engine: the hardware engine
	362	* @req: the request need to be finalized
	363	* @err: error number
	364	*/
	365	void crypto_finalize_aead_request(struct crypto_engine *engine,
	366	struct aead_request *req, int err)
	367	{
	368	return crypto_finalize_request(engine, &req->base, err);
	369	}
	370	EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);
735d37b5	371
218d1cc1 CL	372	/**
	373	* crypto_finalize_akcipher_request - finalize one akcipher_request if
	374	* the request is done
	375	* @engine: the hardware engine
	376	* @req: the request need to be finalized
	377	* @err: error number
	378	*/
	379	void crypto_finalize_akcipher_request(struct crypto_engine *engine,
	380	struct akcipher_request *req, int err)
	381	{
	382	return crypto_finalize_request(engine, &req->base, err);
4cba7cf0	383	}
218d1cc1	384	EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);
4cba7cf0 CL	385
4cba7cf0 CL	386	/**
218d1cc1 CL	387	* crypto_finalize_hash_request - finalize one ahash_request if
218d1cc1 CL	388	* the request is done
4cba7cf0 CL	389	* @engine: the hardware engine
	390	* @req: the request need to be finalized
	391	* @err: error number
	392	*/
	393	void crypto_finalize_hash_request(struct crypto_engine *engine,
	394	struct ahash_request *req, int err)
	395	{
218d1cc1	396	return crypto_finalize_request(engine, &req->base, err);
735d37b5	397	}
4cba7cf0	398	EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
735d37b5	399
1730c5aa PK	400	/**
	401	* crypto_finalize_kpp_request - finalize one kpp_request if the request is done
	402	* @engine: the hardware engine
	403	* @req: the request need to be finalized
	404	* @err: error number
	405	*/
	406	void crypto_finalize_kpp_request(struct crypto_engine *engine,
	407	struct kpp_request *req, int err)
	408	{
	409	return crypto_finalize_request(engine, &req->base, err);
	410	}
	411	EXPORT_SYMBOL_GPL(crypto_finalize_kpp_request);
	412
218d1cc1 CL	413	/**
	414	* crypto_finalize_skcipher_request - finalize one skcipher_request if
	415	* the request is done
	416	* @engine: the hardware engine
	417	* @req: the request need to be finalized
	418	* @err: error number
	419	*/
	420	void crypto_finalize_skcipher_request(struct crypto_engine *engine,
	421	struct skcipher_request *req, int err)
	422	{
	423	return crypto_finalize_request(engine, &req->base, err);
	424	}
	425	EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);
	426
735d37b5 BW	427	/**
	428	* crypto_engine_start - start the hardware engine
	429	* @engine: the hardware engine need to be started
	430	*
	431	* Return 0 on success, else on fail.
	432	*/
	433	int crypto_engine_start(struct crypto_engine *engine)
	434	{
	435	unsigned long flags;
	436
	437	spin_lock_irqsave(&engine->queue_lock, flags);
	438
	439	if (engine->running \|\| engine->busy) {
	440	spin_unlock_irqrestore(&engine->queue_lock, flags);
	441	return -EBUSY;
	442	}
	443
	444	engine->running = true;
	445	spin_unlock_irqrestore(&engine->queue_lock, flags);
	446
c4ca2b0b	447	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	448
	449	return 0;
	450	}
	451	EXPORT_SYMBOL_GPL(crypto_engine_start);
	452
	453	/**
	454	* crypto_engine_stop - stop the hardware engine
	455	* @engine: the hardware engine need to be stopped
	456	*
	457	* Return 0 on success, else on fail.
	458	*/
	459	int crypto_engine_stop(struct crypto_engine *engine)
	460	{
	461	unsigned long flags;
4cba7cf0	462	unsigned int limit = 500;
735d37b5 BW	463	int ret = 0;
	464
	465	spin_lock_irqsave(&engine->queue_lock, flags);
	466
	467	/*
	468	* If the engine queue is not empty or the engine is on busy state,
	469	* we need to wait for a while to pump the requests of engine queue.
	470	*/
	471	while ((crypto_queue_len(&engine->queue) \|\| engine->busy) && limit--) {
	472	spin_unlock_irqrestore(&engine->queue_lock, flags);
	473	msleep(20);
	474	spin_lock_irqsave(&engine->queue_lock, flags);
	475	}
	476
	477	if (crypto_queue_len(&engine->queue) \|\| engine->busy)
	478	ret = -EBUSY;
	479	else
	480	engine->running = false;
	481
	482	spin_unlock_irqrestore(&engine->queue_lock, flags);
	483
	484	if (ret)
88d58ef8	485	dev_warn(engine->dev, "could not stop engine\n");
735d37b5 BW	486
	487	return ret;
	488	}
	489	EXPORT_SYMBOL_GPL(crypto_engine_stop);
	490
	491	/**
6a89f492 IP	492	* crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure
	493	* and initialize it by setting the maximum number of entries in the software
	494	* crypto-engine queue.
735d37b5	495	* @dev: the device attached with one hardware engine
6a89f492	496	* @retry_support: whether hardware has support for retry mechanism
40a3af45	497	* @cbk_do_batch: pointer to a callback function to be invoked when executing
8d908226 IP	498	* a batch of requests.
	499	* This has the form:
	500	* callback(struct crypto_engine *engine)
	501	* where:
42a9a08b	502	* engine: the crypto engine structure.
735d37b5	503	* @rt: whether this queue is set to run as a realtime task
6a89f492	504	* @qlen: maximum size of the crypto-engine queue
735d37b5 BW	505	*
	506	* This must be called from context that can sleep.
	507	* Return: the crypto engine structure on success, else NULL.
	508	*/
6a89f492 IP	509	struct crypto_engine crypto_engine_alloc_init_and_set(struct device dev,
6a89f492 IP	510	bool retry_support,
8d908226	511	int (cbk_do_batch)(struct crypto_engine engine),
6a89f492	512	bool rt, int qlen)
735d37b5	513	{
735d37b5 BW	514	struct crypto_engine *engine;
	515
	516	if (!dev)
	517	return NULL;
	518
	519	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
	520	if (!engine)
	521	return NULL;
	522
88d58ef8	523	engine->dev = dev;
735d37b5 BW	524	engine->rt = rt;
	525	engine->running = false;
	526	engine->busy = false;
	527	engine->idling = false;
6a89f492	528	engine->retry_support = retry_support;
735d37b5	529	engine->priv_data = dev;
8d908226 IP	530	/*
	531	* Batch requests is possible only if
	532	* hardware has support for retry mechanism.
	533	*/
	534	engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;
	535
735d37b5 BW	536	snprintf(engine->name, sizeof(engine->name),
	537	"%s-engine", dev_name(dev));
	538
6a89f492	539	crypto_init_queue(&engine->queue, qlen);
735d37b5 BW	540	spin_lock_init(&engine->queue_lock);
735d37b5 BW	541
c4ca2b0b PM	542	engine->kworker = kthread_create_worker(0, "%s", engine->name);
c4ca2b0b PM	543	if (IS_ERR(engine->kworker)) {
735d37b5 BW	544	dev_err(dev, "failed to create crypto request pump task\n");
	545	return NULL;
	546	}
3989144f	547	kthread_init_work(&engine->pump_requests, crypto_pump_work);
735d37b5 BW	548
	549	if (engine->rt) {
	550	dev_info(dev, "will run requests pump with realtime priority\n");
dbc6d0d5	551	sched_set_fifo(engine->kworker->task);
735d37b5 BW	552	}
	553
	554	return engine;
	555	}
6a89f492 IP	556	EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set);
	557
	558	/**
	559	* crypto_engine_alloc_init - allocate crypto hardware engine structure and
	560	* initialize it.
	561	* @dev: the device attached with one hardware engine
	562	* @rt: whether this queue is set to run as a realtime task
	563	*
	564	* This must be called from context that can sleep.
	565	* Return: the crypto engine structure on success, else NULL.
	566	*/
	567	struct crypto_engine crypto_engine_alloc_init(struct device dev, bool rt)
	568	{
8d908226	569	return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,
6a89f492 IP	570	CRYPTO_ENGINE_MAX_QLEN);
6a89f492 IP	571	}
735d37b5 BW	572	EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
	573
	574	/**
	575	* crypto_engine_exit - free the resources of hardware engine when exit
	576	* @engine: the hardware engine need to be freed
	577	*
	578	* Return 0 for success.
	579	*/
	580	int crypto_engine_exit(struct crypto_engine *engine)
	581	{
	582	int ret;
	583
	584	ret = crypto_engine_stop(engine);
	585	if (ret)
	586	return ret;
	587
c4ca2b0b	588	kthread_destroy_worker(engine->kworker);
735d37b5 BW	589
	590	return 0;
	591	}
	592	EXPORT_SYMBOL_GPL(crypto_engine_exit);
	593
	594	MODULE_LICENSE("GPL");
	595	MODULE_DESCRIPTION("Crypto hardware engine framework");