[linux-2.6-block.git] / drivers / crypto / virtio / virtio_crypto_algs.c

 /* Algorithms supported by virtio crypto device
  *
  * Authors: Gonglei <arei.gonglei@huawei.com>
  *
  * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */

#include <linux/scatterlist.h>
#include <crypto/algapi.h>
#include <linux/err.h>
#include <crypto/scatterwalk.h>
#include <linux/atomic.h>

#include <uapi/linux/virtio_crypto.h>
#include "virtio_crypto_common.h"


struct virtio_crypto_ablkcipher_ctx {
	struct crypto_engine_ctx enginectx;
	struct virtio_crypto *vcrypto;
	struct crypto_tfm *tfm;

	struct virtio_crypto_sym_session_info enc_sess_info;
	struct virtio_crypto_sym_session_info dec_sess_info;
};

struct virtio_crypto_sym_request {
	struct virtio_crypto_request base;

	/* Cipher or aead */
	uint32_t type;
	struct virtio_crypto_ablkcipher_ctx *ablkcipher_ctx;
	struct ablkcipher_request *ablkcipher_req;
	uint8_t *iv;
	/* Encryption? */
	bool encrypt;
};

/*
 * The algs_lock protects the below global virtio_crypto_active_devs
 * and crypto algorithms registion.
 */
static DEFINE_MUTEX(algs_lock);
static unsigned int virtio_crypto_active_devs;
static void virtio_crypto_ablkcipher_finalize_req(
	struct virtio_crypto_sym_request *vc_sym_req,
	struct ablkcipher_request *req,
	int err);

static void virtio_crypto_dataq_sym_callback
		(struct virtio_crypto_request *vc_req, int len)
{
	struct virtio_crypto_sym_request *vc_sym_req =
		container_of(vc_req, struct virtio_crypto_sym_request, base);
	struct ablkcipher_request *ablk_req;
	int error;

	/* Finish the encrypt or decrypt process */
	if (vc_sym_req->type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
		switch (vc_req->status) {
		case VIRTIO_CRYPTO_OK:
			error = 0;
			break;
		case VIRTIO_CRYPTO_INVSESS:
		case VIRTIO_CRYPTO_ERR:
			error = -EINVAL;
			break;
		case VIRTIO_CRYPTO_BADMSG:
			error = -EBADMSG;
			break;
		default:
			error = -EIO;
			break;
		}
		ablk_req = vc_sym_req->ablkcipher_req;
		virtio_crypto_ablkcipher_finalize_req(vc_sym_req,
							ablk_req, error);
	}
}

static u64 virtio_crypto_alg_sg_nents_length(struct scatterlist *sg)
{
	u64 total = 0;

	for (total = 0; sg; sg = sg_next(sg))
		total += sg->length;

	return total;
}

static int
virtio_crypto_alg_validate_key(int key_len, uint32_t *alg)
{
	switch (key_len) {
	case AES_KEYSIZE_128:
	case AES_KEYSIZE_192:
	case AES_KEYSIZE_256:
		*alg = VIRTIO_CRYPTO_CIPHER_AES_CBC;
		break;
	default:
		pr_err("virtio_crypto: Unsupported key length: %d\n",
			key_len);
		return -EINVAL;
	}
	return 0;
}

static int virtio_crypto_alg_ablkcipher_init_session(
		struct virtio_crypto_ablkcipher_ctx *ctx,
		uint32_t alg, const uint8_t *key,
		unsigned int keylen,
		int encrypt)
{
	struct scatterlist outhdr, key_sg, inhdr, *sgs[3];
	unsigned int tmp;
	struct virtio_crypto *vcrypto = ctx->vcrypto;
	int op = encrypt ? VIRTIO_CRYPTO_OP_ENCRYPT : VIRTIO_CRYPTO_OP_DECRYPT;
	int err;
	unsigned int num_out = 0, num_in = 0;

	/*
	 * Avoid to do DMA from the stack, switch to using
	 * dynamically-allocated for the key
	 */
	uint8_t *cipher_key = kmalloc(keylen, GFP_ATOMIC);

	if (!cipher_key)
		return -ENOMEM;

	memcpy(cipher_key, key, keylen);

	spin_lock(&vcrypto->ctrl_lock);
	/* Pad ctrl header */
	vcrypto->ctrl.header.opcode =
		cpu_to_le32(VIRTIO_CRYPTO_CIPHER_CREATE_SESSION);
	vcrypto->ctrl.header.algo = cpu_to_le32(alg);
	/* Set the default dataqueue id to 0 */
	vcrypto->ctrl.header.queue_id = 0;

	vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR);
	/* Pad cipher's parameters */
	vcrypto->ctrl.u.sym_create_session.op_type =
		cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER);
	vcrypto->ctrl.u.sym_create_session.u.cipher.para.algo =
		vcrypto->ctrl.header.algo;
	vcrypto->ctrl.u.sym_create_session.u.cipher.para.keylen =
		cpu_to_le32(keylen);
	vcrypto->ctrl.u.sym_create_session.u.cipher.para.op =
		cpu_to_le32(op);

	sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
	sgs[num_out++] = &outhdr;

	/* Set key */
	sg_init_one(&key_sg, cipher_key, keylen);
	sgs[num_out++] = &key_sg;

	/* Return status and session id back */
	sg_init_one(&inhdr, &vcrypto->input, sizeof(vcrypto->input));
	sgs[num_out + num_in++] = &inhdr;

	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out,
				num_in, vcrypto, GFP_ATOMIC);
	if (err < 0) {
		spin_unlock(&vcrypto->ctrl_lock);
		kzfree(cipher_key);
		return err;
	}
	virtqueue_kick(vcrypto->ctrl_vq);

	/*
	 * Trapping into the hypervisor, so the request should be
	 * handled immediately.
	 */
	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) &&
	       !virtqueue_is_broken(vcrypto->ctrl_vq))
		cpu_relax();

	if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) {
		spin_unlock(&vcrypto->ctrl_lock);
		pr_err("virtio_crypto: Create session failed status: %u\n",
			le32_to_cpu(vcrypto->input.status));
		kzfree(cipher_key);
		return -EINVAL;
	}

	if (encrypt)
		ctx->enc_sess_info.session_id =
			le64_to_cpu(vcrypto->input.session_id);
	else
		ctx->dec_sess_info.session_id =
			le64_to_cpu(vcrypto->input.session_id);

	spin_unlock(&vcrypto->ctrl_lock);

	kzfree(cipher_key);
	return 0;
}

static int virtio_crypto_alg_ablkcipher_close_session(
		struct virtio_crypto_ablkcipher_ctx *ctx,
		int encrypt)
{
	struct scatterlist outhdr, status_sg, *sgs[2];
	unsigned int tmp;
	struct virtio_crypto_destroy_session_req *destroy_session;
	struct virtio_crypto *vcrypto = ctx->vcrypto;
	int err;
	unsigned int num_out = 0, num_in = 0;

	spin_lock(&vcrypto->ctrl_lock);
	vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR;
	/* Pad ctrl header */
	vcrypto->ctrl.header.opcode =
		cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION);
	/* Set the default virtqueue id to 0 */
	vcrypto->ctrl.header.queue_id = 0;

	destroy_session = &vcrypto->ctrl.u.destroy_session;

	if (encrypt)
		destroy_session->session_id =
			cpu_to_le64(ctx->enc_sess_info.session_id);
	else
		destroy_session->session_id =
			cpu_to_le64(ctx->dec_sess_info.session_id);

	sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
	sgs[num_out++] = &outhdr;

	/* Return status and session id back */
	sg_init_one(&status_sg, &vcrypto->ctrl_status.status,
		sizeof(vcrypto->ctrl_status.status));
	sgs[num_out + num_in++] = &status_sg;

	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out,
			num_in, vcrypto, GFP_ATOMIC);
	if (err < 0) {
		spin_unlock(&vcrypto->ctrl_lock);
		return err;
	}
	virtqueue_kick(vcrypto->ctrl_vq);

	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) &&
	       !virtqueue_is_broken(vcrypto->ctrl_vq))
		cpu_relax();

	if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) {
		spin_unlock(&vcrypto->ctrl_lock);
		pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n",
			vcrypto->ctrl_status.status,
			destroy_session->session_id);

		return -EINVAL;
	}
	spin_unlock(&vcrypto->ctrl_lock);

	return 0;
}

static int virtio_crypto_alg_ablkcipher_init_sessions(
		struct virtio_crypto_ablkcipher_ctx *ctx,
		const uint8_t *key, unsigned int keylen)
{
	uint32_t alg;
	int ret;
	struct virtio_crypto *vcrypto = ctx->vcrypto;

	if (keylen > vcrypto->max_cipher_key_len) {
		pr_err("virtio_crypto: the key is too long\n");
		goto bad_key;
	}

	if (virtio_crypto_alg_validate_key(keylen, &alg))
		goto bad_key;

	/* Create encryption session */
	ret = virtio_crypto_alg_ablkcipher_init_session(ctx,
			alg, key, keylen, 1);
	if (ret)
		return ret;
	/* Create decryption session */
	ret = virtio_crypto_alg_ablkcipher_init_session(ctx,
			alg, key, keylen, 0);
	if (ret) {
		virtio_crypto_alg_ablkcipher_close_session(ctx, 1);
		return ret;
	}
	return 0;

bad_key:
	crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
	return -EINVAL;
}

/* Note: kernel crypto API realization */
static int virtio_crypto_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
					 const uint8_t *key,
					 unsigned int keylen)
{
	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
	int ret;

	if (!ctx->vcrypto) {
		/* New key */
		int node = virtio_crypto_get_current_node();
		struct virtio_crypto *vcrypto =
				      virtcrypto_get_dev_node(node);
		if (!vcrypto) {
			pr_err("virtio_crypto: Could not find a virtio device in the system\n");
			return -ENODEV;
		}

		ctx->vcrypto = vcrypto;
	} else {
		/* Rekeying, we should close the created sessions previously */
		virtio_crypto_alg_ablkcipher_close_session(ctx, 1);
		virtio_crypto_alg_ablkcipher_close_session(ctx, 0);
	}

	ret = virtio_crypto_alg_ablkcipher_init_sessions(ctx, key, keylen);
	if (ret) {
		virtcrypto_dev_put(ctx->vcrypto);
		ctx->vcrypto = NULL;

		return ret;
	}

	return 0;
}

static int
__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req,
		struct ablkcipher_request *req,
		struct data_queue *data_vq)
{
	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
	struct virtio_crypto_ablkcipher_ctx *ctx = vc_sym_req->ablkcipher_ctx;
	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
	unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
	struct virtio_crypto *vcrypto = ctx->vcrypto;
	struct virtio_crypto_op_data_req *req_data;
	int src_nents, dst_nents;
	int err;
	unsigned long flags;
	struct scatterlist outhdr, iv_sg, status_sg, **sgs;
	int i;
	u64 dst_len;
	unsigned int num_out = 0, num_in = 0;
	int sg_total;
	uint8_t *iv;

	src_nents = sg_nents_for_len(req->src, req->nbytes);
	dst_nents = sg_nents(req->dst);

	pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n",
			src_nents, dst_nents);

	/* Why 3?  outhdr + iv + inhdr */
	sg_total = src_nents + dst_nents + 3;
	sgs = kcalloc_node(sg_total, sizeof(*sgs), GFP_ATOMIC,
				dev_to_node(&vcrypto->vdev->dev));
	if (!sgs)
		return -ENOMEM;

	req_data = kzalloc_node(sizeof(*req_data), GFP_ATOMIC,
				dev_to_node(&vcrypto->vdev->dev));
	if (!req_data) {
		kfree(sgs);
		return -ENOMEM;
	}

	vc_req->req_data = req_data;
	vc_sym_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER;
	/* Head of operation */
	if (vc_sym_req->encrypt) {
		req_data->header.session_id =
			cpu_to_le64(ctx->enc_sess_info.session_id);
		req_data->header.opcode =
			cpu_to_le32(VIRTIO_CRYPTO_CIPHER_ENCRYPT);
	} else {
		req_data->header.session_id =
			cpu_to_le64(ctx->dec_sess_info.session_id);
	    req_data->header.opcode =
			cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DECRYPT);
	}
	req_data->u.sym_req.op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER);
	req_data->u.sym_req.u.cipher.para.iv_len = cpu_to_le32(ivsize);
	req_data->u.sym_req.u.cipher.para.src_data_len =
			cpu_to_le32(req->nbytes);

	dst_len = virtio_crypto_alg_sg_nents_length(req->dst);
	if (unlikely(dst_len > U32_MAX)) {
		pr_err("virtio_crypto: The dst_len is beyond U32_MAX\n");
		err = -EINVAL;
		goto free;
	}

	pr_debug("virtio_crypto: src_len: %u, dst_len: %llu\n",
			req->nbytes, dst_len);

	if (unlikely(req->nbytes + dst_len + ivsize +
		sizeof(vc_req->status) > vcrypto->max_size)) {
		pr_err("virtio_crypto: The length is too big\n");
		err = -EINVAL;
		goto free;
	}

	req_data->u.sym_req.u.cipher.para.dst_data_len =
			cpu_to_le32((uint32_t)dst_len);

	/* Outhdr */
	sg_init_one(&outhdr, req_data, sizeof(*req_data));
	sgs[num_out++] = &outhdr;

	/* IV */

	/*
	 * Avoid to do DMA from the stack, switch to using
	 * dynamically-allocated for the IV
	 */
	iv = kzalloc_node(ivsize, GFP_ATOMIC,
				dev_to_node(&vcrypto->vdev->dev));
	if (!iv) {
		err = -ENOMEM;
		goto free;
	}
	memcpy(iv, req->info, ivsize);
	sg_init_one(&iv_sg, iv, ivsize);
	sgs[num_out++] = &iv_sg;
	vc_sym_req->iv = iv;

	/* Source data */
	for (i = 0; i < src_nents; i++)
		sgs[num_out++] = &req->src[i];

	/* Destination data */
	for (i = 0; i < dst_nents; i++)
		sgs[num_out + num_in++] = &req->dst[i];

	/* Status */
	sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status));
	sgs[num_out + num_in++] = &status_sg;

	vc_req->sgs = sgs;

	spin_lock_irqsave(&data_vq->lock, flags);
	err = virtqueue_add_sgs(data_vq->vq, sgs, num_out,
				num_in, vc_req, GFP_ATOMIC);
	virtqueue_kick(data_vq->vq);
	spin_unlock_irqrestore(&data_vq->lock, flags);
	if (unlikely(err < 0))
		goto free_iv;

	return 0;

free_iv:
	kzfree(iv);
free:
	kzfree(req_data);
	kfree(sgs);
	return err;
}

static int virtio_crypto_ablkcipher_encrypt(struct ablkcipher_request *req)
{
	struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
	struct virtio_crypto_sym_request *vc_sym_req =
				ablkcipher_request_ctx(req);
	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
	struct virtio_crypto *vcrypto = ctx->vcrypto;
	/* Use the first data virtqueue as default */
	struct data_queue *data_vq = &vcrypto->data_vq[0];

	vc_req->dataq = data_vq;
	vc_req->alg_cb = virtio_crypto_dataq_sym_callback;
	vc_sym_req->ablkcipher_ctx = ctx;
	vc_sym_req->ablkcipher_req = req;
	vc_sym_req->encrypt = true;

	return crypto_transfer_ablkcipher_request_to_engine(data_vq->engine, req);
}

static int virtio_crypto_ablkcipher_decrypt(struct ablkcipher_request *req)
{
	struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
	struct virtio_crypto_sym_request *vc_sym_req =
				ablkcipher_request_ctx(req);
	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
	struct virtio_crypto *vcrypto = ctx->vcrypto;
	/* Use the first data virtqueue as default */
	struct data_queue *data_vq = &vcrypto->data_vq[0];

	vc_req->dataq = data_vq;
	vc_req->alg_cb = virtio_crypto_dataq_sym_callback;
	vc_sym_req->ablkcipher_ctx = ctx;
	vc_sym_req->ablkcipher_req = req;
	vc_sym_req->encrypt = false;

	return crypto_transfer_ablkcipher_request_to_engine(data_vq->engine, req);
}

static int virtio_crypto_ablkcipher_init(struct crypto_tfm *tfm)
{
	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);

	tfm->crt_ablkcipher.reqsize = sizeof(struct virtio_crypto_sym_request);
	ctx->tfm = tfm;

	ctx->enginectx.op.do_one_request = virtio_crypto_ablkcipher_crypt_req;
	ctx->enginectx.op.prepare_request = NULL;
	ctx->enginectx.op.unprepare_request = NULL;
	return 0;
}

static void virtio_crypto_ablkcipher_exit(struct crypto_tfm *tfm)
{
	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);

	if (!ctx->vcrypto)
		return;

	virtio_crypto_alg_ablkcipher_close_session(ctx, 1);
	virtio_crypto_alg_ablkcipher_close_session(ctx, 0);
	virtcrypto_dev_put(ctx->vcrypto);
	ctx->vcrypto = NULL;
}

int virtio_crypto_ablkcipher_crypt_req(
	struct crypto_engine *engine, void *vreq)
{
	struct ablkcipher_request *req = container_of(vreq, struct ablkcipher_request, base);
	struct virtio_crypto_sym_request *vc_sym_req =
				ablkcipher_request_ctx(req);
	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
	struct data_queue *data_vq = vc_req->dataq;
	int ret;

	ret = __virtio_crypto_ablkcipher_do_req(vc_sym_req, req, data_vq);
	if (ret < 0)
		return ret;

	virtqueue_kick(data_vq->vq);

	return 0;
}

static void virtio_crypto_ablkcipher_finalize_req(
	struct virtio_crypto_sym_request *vc_sym_req,
	struct ablkcipher_request *req,
	int err)
{
	crypto_finalize_ablkcipher_request(vc_sym_req->base.dataq->engine,
					   req, err);
	kzfree(vc_sym_req->iv);
	virtcrypto_clear_request(&vc_sym_req->base);
}

static struct crypto_alg virtio_crypto_algs[] = { {
	.cra_name = "cbc(aes)",
	.cra_driver_name = "virtio_crypto_aes_cbc",
	.cra_priority = 150,
	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize = AES_BLOCK_SIZE,
	.cra_ctxsize  = sizeof(struct virtio_crypto_ablkcipher_ctx),
	.cra_alignmask = 0,
	.cra_module = THIS_MODULE,
	.cra_type = &crypto_ablkcipher_type,
	.cra_init = virtio_crypto_ablkcipher_init,
	.cra_exit = virtio_crypto_ablkcipher_exit,
	.cra_u = {
	   .ablkcipher = {
			.setkey = virtio_crypto_ablkcipher_setkey,
			.decrypt = virtio_crypto_ablkcipher_decrypt,
			.encrypt = virtio_crypto_ablkcipher_encrypt,
			.min_keysize = AES_MIN_KEY_SIZE,
			.max_keysize = AES_MAX_KEY_SIZE,
			.ivsize = AES_BLOCK_SIZE,
		},
	},
} };

int virtio_crypto_algs_register(void)
{
	int ret = 0;

	mutex_lock(&algs_lock);
	if (++virtio_crypto_active_devs != 1)
		goto unlock;

	ret = crypto_register_algs(virtio_crypto_algs,
			ARRAY_SIZE(virtio_crypto_algs));
	if (ret)
		virtio_crypto_active_devs--;

unlock:
	mutex_unlock(&algs_lock);
	return ret;
}

void virtio_crypto_algs_unregister(void)
{
	mutex_lock(&algs_lock);
	if (--virtio_crypto_active_devs != 0)
		goto unlock;

	crypto_unregister_algs(virtio_crypto_algs,
			ARRAY_SIZE(virtio_crypto_algs));

unlock:
	mutex_unlock(&algs_lock);
}
Commit	Line	Data
dbaf0624 G	1	/* Algorithms supported by virtio crypto device
	2	*
	3	* Authors: Gonglei <arei.gonglei@huawei.com>
	4	*
	5	* Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD.
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License as published by
	9	* the Free Software Foundation; either version 2 of the License, or
	10	* (at your option) any later version.
	11	*
	12	* This program is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	* GNU General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU General Public License
	18	* along with this program; if not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
	21	#include <linux/scatterlist.h>
	22	#include <crypto/algapi.h>
	23	#include <linux/err.h>
	24	#include <crypto/scatterwalk.h>
	25	#include <linux/atomic.h>
	26
	27	#include <uapi/linux/virtio_crypto.h>
	28	#include "virtio_crypto_common.h"
	29
d31e7123 ZX	30
d31e7123 ZX	31	struct virtio_crypto_ablkcipher_ctx {
67189375	32	struct crypto_engine_ctx enginectx;
d31e7123 ZX	33	struct virtio_crypto *vcrypto;
	34	struct crypto_tfm *tfm;
	35
	36	struct virtio_crypto_sym_session_info enc_sess_info;
	37	struct virtio_crypto_sym_session_info dec_sess_info;
	38	};
	39
	40	struct virtio_crypto_sym_request {
	41	struct virtio_crypto_request base;
	42
	43	/* Cipher or aead */
	44	uint32_t type;
	45	struct virtio_crypto_ablkcipher_ctx *ablkcipher_ctx;
	46	struct ablkcipher_request *ablkcipher_req;
	47	uint8_t *iv;
	48	/* Encryption? */
	49	bool encrypt;
	50	};
	51
dbaf0624 G	52	/*
	53	* The algs_lock protects the below global virtio_crypto_active_devs
	54	* and crypto algorithms registion.
	55	*/
	56	static DEFINE_MUTEX(algs_lock);
	57	static unsigned int virtio_crypto_active_devs;
d31e7123 ZX	58	static void virtio_crypto_ablkcipher_finalize_req(
	59	struct virtio_crypto_sym_request *vc_sym_req,
	60	struct ablkcipher_request *req,
	61	int err);
	62
	63	static void virtio_crypto_dataq_sym_callback
	64	(struct virtio_crypto_request *vc_req, int len)
	65	{
	66	struct virtio_crypto_sym_request *vc_sym_req =
	67	container_of(vc_req, struct virtio_crypto_sym_request, base);
	68	struct ablkcipher_request *ablk_req;
	69	int error;
	70
	71	/* Finish the encrypt or decrypt process */
	72	if (vc_sym_req->type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
	73	switch (vc_req->status) {
	74	case VIRTIO_CRYPTO_OK:
	75	error = 0;
	76	break;
	77	case VIRTIO_CRYPTO_INVSESS:
	78	case VIRTIO_CRYPTO_ERR:
	79	error = -EINVAL;
	80	break;
	81	case VIRTIO_CRYPTO_BADMSG:
	82	error = -EBADMSG;
	83	break;
	84	default:
	85	error = -EIO;
	86	break;
	87	}
	88	ablk_req = vc_sym_req->ablkcipher_req;
	89	virtio_crypto_ablkcipher_finalize_req(vc_sym_req,
	90	ablk_req, error);
	91	}
	92	}
dbaf0624 G	93
	94	static u64 virtio_crypto_alg_sg_nents_length(struct scatterlist *sg)
	95	{
	96	u64 total = 0;
	97
	98	for (total = 0; sg; sg = sg_next(sg))
	99	total += sg->length;
	100
	101	return total;
	102	}
	103
	104	static int
	105	virtio_crypto_alg_validate_key(int key_len, uint32_t *alg)
	106	{
	107	switch (key_len) {
	108	case AES_KEYSIZE_128:
	109	case AES_KEYSIZE_192:
	110	case AES_KEYSIZE_256:
	111	*alg = VIRTIO_CRYPTO_CIPHER_AES_CBC;
	112	break;
	113	default:
	114	pr_err("virtio_crypto: Unsupported key length: %d\n",
	115	key_len);
	116	return -EINVAL;
	117	}
	118	return 0;
	119	}
	120
	121	static int virtio_crypto_alg_ablkcipher_init_session(
	122	struct virtio_crypto_ablkcipher_ctx *ctx,
	123	uint32_t alg, const uint8_t *key,
	124	unsigned int keylen,
	125	int encrypt)
	126	{
	127	struct scatterlist outhdr, key_sg, inhdr, *sgs[3];
	128	unsigned int tmp;
	129	struct virtio_crypto *vcrypto = ctx->vcrypto;
	130	int op = encrypt ? VIRTIO_CRYPTO_OP_ENCRYPT : VIRTIO_CRYPTO_OP_DECRYPT;
	131	int err;
	132	unsigned int num_out = 0, num_in = 0;
	133
	134	/*
	135	* Avoid to do DMA from the stack, switch to using
	136	* dynamically-allocated for the key
	137	*/
	138	uint8_t *cipher_key = kmalloc(keylen, GFP_ATOMIC);
	139
	140	if (!cipher_key)
	141	return -ENOMEM;
	142
	143	memcpy(cipher_key, key, keylen);
	144
	145	spin_lock(&vcrypto->ctrl_lock);
	146	/* Pad ctrl header */
	147	vcrypto->ctrl.header.opcode =
	148	cpu_to_le32(VIRTIO_CRYPTO_CIPHER_CREATE_SESSION);
	149	vcrypto->ctrl.header.algo = cpu_to_le32(alg);
	150	/* Set the default dataqueue id to 0 */
	151	vcrypto->ctrl.header.queue_id = 0;
	152
	153	vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR);
	154	/* Pad cipher's parameters */
	155	vcrypto->ctrl.u.sym_create_session.op_type =
	156	cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER);
157	vcrypto->ctrl.u.sym_create_session.u.cipher.para.algo =
158	vcrypto->ctrl.header.algo;
159	vcrypto->ctrl.u.sym_create_session.u.cipher.para.keylen =
160	cpu_to_le32(keylen);
161	vcrypto->ctrl.u.sym_create_session.u.cipher.para.op =
162	cpu_to_le32(op);
163
164	sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
165	sgs[num_out++] = &outhdr;
166
167	/* Set key */
168	sg_init_one(&key_sg, cipher_key, keylen);
169	sgs[num_out++] = &key_sg;
170
171	/* Return status and session id back */
172	sg_init_one(&inhdr, &vcrypto->input, sizeof(vcrypto->input));
173	sgs[num_out + num_in++] = &inhdr;
174
175	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out,
176	num_in, vcrypto, GFP_ATOMIC);
177	if (err < 0) {
178	spin_unlock(&vcrypto->ctrl_lock);
179	kzfree(cipher_key);
180	return err;
181	}
182	virtqueue_kick(vcrypto->ctrl_vq);
183
184	/*
185	* Trapping into the hypervisor, so the request should be
186	* handled immediately.
187	*/
188	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) &&
189	!virtqueue_is_broken(vcrypto->ctrl_vq))
190	cpu_relax();
191
192	if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) {
193	spin_unlock(&vcrypto->ctrl_lock);
194	pr_err("virtio_crypto: Create session failed status: %u\n",
195	le32_to_cpu(vcrypto->input.status));
196	kzfree(cipher_key);
197	return -EINVAL;
198	}
199
200	if (encrypt)
201	ctx->enc_sess_info.session_id =
202	le64_to_cpu(vcrypto->input.session_id);
203	else
204	ctx->dec_sess_info.session_id =
205	le64_to_cpu(vcrypto->input.session_id);
206
207	spin_unlock(&vcrypto->ctrl_lock);
208
209	kzfree(cipher_key);
210	return 0;
211	}
212
213	static int virtio_crypto_alg_ablkcipher_close_session(
214	struct virtio_crypto_ablkcipher_ctx *ctx,
215	int encrypt)
216	{
217	struct scatterlist outhdr, status_sg, *sgs[2];
218	unsigned int tmp;
219	struct virtio_crypto_destroy_session_req *destroy_session;
220	struct virtio_crypto *vcrypto = ctx->vcrypto;
221	int err;
222	unsigned int num_out = 0, num_in = 0;
223
224	spin_lock(&vcrypto->ctrl_lock);
225	vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR;
226	/* Pad ctrl header */
227	vcrypto->ctrl.header.opcode =
228	cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION);
229	/* Set the default virtqueue id to 0 */
230	vcrypto->ctrl.header.queue_id = 0;
231
232	destroy_session = &vcrypto->ctrl.u.destroy_session;
233
234	if (encrypt)
235	destroy_session->session_id =
236	cpu_to_le64(ctx->enc_sess_info.session_id);
237	else
238	destroy_session->session_id =
239	cpu_to_le64(ctx->dec_sess_info.session_id);
240
241	sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
242	sgs[num_out++] = &outhdr;
243
244	/* Return status and session id back */
245	sg_init_one(&status_sg, &vcrypto->ctrl_status.status,
246	sizeof(vcrypto->ctrl_status.status));
247	sgs[num_out + num_in++] = &status_sg;
248
249	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out,
250	num_in, vcrypto, GFP_ATOMIC);
251	if (err < 0) {
252	spin_unlock(&vcrypto->ctrl_lock);
253	return err;
254	}
255	virtqueue_kick(vcrypto->ctrl_vq);
256
257	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) &&
258	!virtqueue_is_broken(vcrypto->ctrl_vq))
259	cpu_relax();
260
261	if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) {
262	spin_unlock(&vcrypto->ctrl_lock);
263	pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n",
264	vcrypto->ctrl_status.status,
265	destroy_session->session_id);
266
267	return -EINVAL;
268	}
269	spin_unlock(&vcrypto->ctrl_lock);
270
271	return 0;
272	}
273
274	static int virtio_crypto_alg_ablkcipher_init_sessions(
275	struct virtio_crypto_ablkcipher_ctx *ctx,
276	const uint8_t *key, unsigned int keylen)
277	{
278	uint32_t alg;
279	int ret;
280	struct virtio_crypto *vcrypto = ctx->vcrypto;
281
282	if (keylen > vcrypto->max_cipher_key_len) {
283	pr_err("virtio_crypto: the key is too long\n");
284	goto bad_key;
285	}
286
287	if (virtio_crypto_alg_validate_key(keylen, &alg))
288	goto bad_key;
289
290	/* Create encryption session */
291	ret = virtio_crypto_alg_ablkcipher_init_session(ctx,
292	alg, key, keylen, 1);
293	if (ret)
294	return ret;
295	/* Create decryption session */
296	ret = virtio_crypto_alg_ablkcipher_init_session(ctx,
297	alg, key, keylen, 0);
298	if (ret) {
299	virtio_crypto_alg_ablkcipher_close_session(ctx, 1);
300	return ret;
301	}
302	return 0;
303
304	bad_key:
305	crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
306	return -EINVAL;
307	}
308
309	/* Note: kernel crypto API realization */
310	static int virtio_crypto_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
311	const uint8_t *key,
312	unsigned int keylen)
313	{
314	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
315	int ret;
316
317	if (!ctx->vcrypto) {
318	/* New key */
319	int node = virtio_crypto_get_current_node();
320	struct virtio_crypto *vcrypto =
321	virtcrypto_get_dev_node(node);
322	if (!vcrypto) {
e02b8b43	323	pr_err("virtio_crypto: Could not find a virtio device in the system\n");
dbaf0624 G	324	return -ENODEV;
	325	}
	326
	327	ctx->vcrypto = vcrypto;
	328	} else {
	329	/* Rekeying, we should close the created sessions previously */
	330	virtio_crypto_alg_ablkcipher_close_session(ctx, 1);
	331	virtio_crypto_alg_ablkcipher_close_session(ctx, 0);
	332	}
	333
	334	ret = virtio_crypto_alg_ablkcipher_init_sessions(ctx, key, keylen);
	335	if (ret) {
	336	virtcrypto_dev_put(ctx->vcrypto);
	337	ctx->vcrypto = NULL;
	338
	339	return ret;
	340	}
	341
	342	return 0;
	343	}
	344
	345	static int
d31e7123	346	__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req,
dbaf0624	347	struct ablkcipher_request *req,
d79b5d0b	348	struct data_queue *data_vq)
dbaf0624 G	349	{
dbaf0624 G	350	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
d31e7123 ZX	351	struct virtio_crypto_ablkcipher_ctx *ctx = vc_sym_req->ablkcipher_ctx;
d31e7123 ZX	352	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
dbaf0624	353	unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
dbaf0624 G	354	struct virtio_crypto *vcrypto = ctx->vcrypto;
	355	struct virtio_crypto_op_data_req *req_data;
	356	int src_nents, dst_nents;
	357	int err;
	358	unsigned long flags;
	359	struct scatterlist outhdr, iv_sg, status_sg, **sgs;
	360	int i;
	361	u64 dst_len;
	362	unsigned int num_out = 0, num_in = 0;
	363	int sg_total;
	364	uint8_t *iv;
	365
	366	src_nents = sg_nents_for_len(req->src, req->nbytes);
	367	dst_nents = sg_nents(req->dst);
	368
	369	pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n",
	370	src_nents, dst_nents);
	371
	372	/* Why 3? outhdr + iv + inhdr */
	373	sg_total = src_nents + dst_nents + 3;
590b5b7d	374	sgs = kcalloc_node(sg_total, sizeof(*sgs), GFP_ATOMIC,
dbaf0624 G	375	dev_to_node(&vcrypto->vdev->dev));
	376	if (!sgs)
	377	return -ENOMEM;
	378
	379	req_data = kzalloc_node(sizeof(*req_data), GFP_ATOMIC,
	380	dev_to_node(&vcrypto->vdev->dev));
	381	if (!req_data) {
	382	kfree(sgs);
	383	return -ENOMEM;
	384	}
	385
	386	vc_req->req_data = req_data;
d31e7123	387	vc_sym_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER;
dbaf0624	388	/* Head of operation */
d31e7123	389	if (vc_sym_req->encrypt) {
dbaf0624 G	390	req_data->header.session_id =
	391	cpu_to_le64(ctx->enc_sess_info.session_id);
	392	req_data->header.opcode =
	393	cpu_to_le32(VIRTIO_CRYPTO_CIPHER_ENCRYPT);
	394	} else {
	395	req_data->header.session_id =
	396	cpu_to_le64(ctx->dec_sess_info.session_id);
	397	req_data->header.opcode =
	398	cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DECRYPT);
	399	}
	400	req_data->u.sym_req.op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER);
	401	req_data->u.sym_req.u.cipher.para.iv_len = cpu_to_le32(ivsize);
	402	req_data->u.sym_req.u.cipher.para.src_data_len =
	403	cpu_to_le32(req->nbytes);
	404
	405	dst_len = virtio_crypto_alg_sg_nents_length(req->dst);
	406	if (unlikely(dst_len > U32_MAX)) {
	407	pr_err("virtio_crypto: The dst_len is beyond U32_MAX\n");
	408	err = -EINVAL;
	409	goto free;
	410	}
	411
	412	pr_debug("virtio_crypto: src_len: %u, dst_len: %llu\n",
	413	req->nbytes, dst_len);
	414
	415	if (unlikely(req->nbytes + dst_len + ivsize +
	416	sizeof(vc_req->status) > vcrypto->max_size)) {
	417	pr_err("virtio_crypto: The length is too big\n");
	418	err = -EINVAL;
	419	goto free;
	420	}
	421
	422	req_data->u.sym_req.u.cipher.para.dst_data_len =
	423	cpu_to_le32((uint32_t)dst_len);
	424
	425	/* Outhdr */
	426	sg_init_one(&outhdr, req_data, sizeof(*req_data));
	427	sgs[num_out++] = &outhdr;
	428
	429	/* IV */
	430
	431	/*
	432	* Avoid to do DMA from the stack, switch to using
	433	* dynamically-allocated for the IV
	434	*/
	435	iv = kzalloc_node(ivsize, GFP_ATOMIC,
	436	dev_to_node(&vcrypto->vdev->dev));
	437	if (!iv) {
	438	err = -ENOMEM;
	439	goto free;
	440	}
	441	memcpy(iv, req->info, ivsize);
	442	sg_init_one(&iv_sg, iv, ivsize);
	443	sgs[num_out++] = &iv_sg;
d31e7123	444	vc_sym_req->iv = iv;
dbaf0624 G	445
	446	/* Source data */
	447	for (i = 0; i < src_nents; i++)
	448	sgs[num_out++] = &req->src[i];
	449
	450	/* Destination data */
	451	for (i = 0; i < dst_nents; i++)
	452	sgs[num_out + num_in++] = &req->dst[i];
	453
	454	/* Status */
	455	sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status));
	456	sgs[num_out + num_in++] = &status_sg;
	457
	458	vc_req->sgs = sgs;
	459
	460	spin_lock_irqsave(&data_vq->lock, flags);
	461	err = virtqueue_add_sgs(data_vq->vq, sgs, num_out,
	462	num_in, vc_req, GFP_ATOMIC);
	463	virtqueue_kick(data_vq->vq);
	464	spin_unlock_irqrestore(&data_vq->lock, flags);
	465	if (unlikely(err < 0))
	466	goto free_iv;
	467
	468	return 0;
	469
	470	free_iv:
	471	kzfree(iv);
	472	free:
	473	kzfree(req_data);
	474	kfree(sgs);
	475	return err;
	476	}
	477
	478	static int virtio_crypto_ablkcipher_encrypt(struct ablkcipher_request *req)
	479	{
	480	struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
	481	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
d31e7123 ZX	482	struct virtio_crypto_sym_request *vc_sym_req =
	483	ablkcipher_request_ctx(req);
	484	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
dbaf0624	485	struct virtio_crypto *vcrypto = ctx->vcrypto;
dbaf0624 G	486	/* Use the first data virtqueue as default */
	487	struct data_queue *data_vq = &vcrypto->data_vq[0];
	488
d79b5d0b	489	vc_req->dataq = data_vq;
d31e7123 ZX	490	vc_req->alg_cb = virtio_crypto_dataq_sym_callback;
	491	vc_sym_req->ablkcipher_ctx = ctx;
	492	vc_sym_req->ablkcipher_req = req;
	493	vc_sym_req->encrypt = true;
dbaf0624	494
67189375	495	return crypto_transfer_ablkcipher_request_to_engine(data_vq->engine, req);
dbaf0624 G	496	}
	497
	498	static int virtio_crypto_ablkcipher_decrypt(struct ablkcipher_request *req)
	499	{
	500	struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
	501	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
d31e7123 ZX	502	struct virtio_crypto_sym_request *vc_sym_req =
	503	ablkcipher_request_ctx(req);
	504	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
dbaf0624	505	struct virtio_crypto *vcrypto = ctx->vcrypto;
dbaf0624 G	506	/* Use the first data virtqueue as default */
	507	struct data_queue *data_vq = &vcrypto->data_vq[0];
	508
d79b5d0b	509	vc_req->dataq = data_vq;
d31e7123 ZX	510	vc_req->alg_cb = virtio_crypto_dataq_sym_callback;
	511	vc_sym_req->ablkcipher_ctx = ctx;
	512	vc_sym_req->ablkcipher_req = req;
	513	vc_sym_req->encrypt = false;
dbaf0624	514
67189375	515	return crypto_transfer_ablkcipher_request_to_engine(data_vq->engine, req);
dbaf0624 G	516	}
	517
	518	static int virtio_crypto_ablkcipher_init(struct crypto_tfm *tfm)
	519	{
	520	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
	521
d31e7123	522	tfm->crt_ablkcipher.reqsize = sizeof(struct virtio_crypto_sym_request);
dbaf0624 G	523	ctx->tfm = tfm;
dbaf0624 G	524
67189375 CL	525	ctx->enginectx.op.do_one_request = virtio_crypto_ablkcipher_crypt_req;
	526	ctx->enginectx.op.prepare_request = NULL;
	527	ctx->enginectx.op.unprepare_request = NULL;
dbaf0624 G	528	return 0;
	529	}
	530
	531	static void virtio_crypto_ablkcipher_exit(struct crypto_tfm *tfm)
	532	{
	533	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
	534
	535	if (!ctx->vcrypto)
	536	return;
	537
	538	virtio_crypto_alg_ablkcipher_close_session(ctx, 1);
	539	virtio_crypto_alg_ablkcipher_close_session(ctx, 0);
	540	virtcrypto_dev_put(ctx->vcrypto);
	541	ctx->vcrypto = NULL;
	542	}
	543
d79b5d0b	544	int virtio_crypto_ablkcipher_crypt_req(
67189375	545	struct crypto_engine engine, void vreq)
d79b5d0b	546	{
67189375	547	struct ablkcipher_request *req = container_of(vreq, struct ablkcipher_request, base);
d31e7123 ZX	548	struct virtio_crypto_sym_request *vc_sym_req =
	549	ablkcipher_request_ctx(req);
	550	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
d79b5d0b GA	551	struct data_queue *data_vq = vc_req->dataq;
	552	int ret;
	553
d31e7123	554	ret = __virtio_crypto_ablkcipher_do_req(vc_sym_req, req, data_vq);
d79b5d0b GA	555	if (ret < 0)
	556	return ret;
	557
	558	virtqueue_kick(data_vq->vq);
	559
	560	return 0;
	561	}
	562
d31e7123 ZX	563	static void virtio_crypto_ablkcipher_finalize_req(
d31e7123 ZX	564	struct virtio_crypto_sym_request *vc_sym_req,
d79b5d0b GA	565	struct ablkcipher_request *req,
	566	int err)
	567	{
67189375 CL	568	crypto_finalize_ablkcipher_request(vc_sym_req->base.dataq->engine,
67189375 CL	569	req, err);
d31e7123 ZX	570	kzfree(vc_sym_req->iv);
d31e7123 ZX	571	virtcrypto_clear_request(&vc_sym_req->base);
d79b5d0b GA	572	}
d79b5d0b GA	573
dbaf0624 G	574	static struct crypto_alg virtio_crypto_algs[] = { {
	575	.cra_name = "cbc(aes)",
	576	.cra_driver_name = "virtio_crypto_aes_cbc",
87170961	577	.cra_priority = 150,
dbaf0624 G	578	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	579	.cra_blocksize = AES_BLOCK_SIZE,
	580	.cra_ctxsize = sizeof(struct virtio_crypto_ablkcipher_ctx),
	581	.cra_alignmask = 0,
	582	.cra_module = THIS_MODULE,
	583	.cra_type = &crypto_ablkcipher_type,
	584	.cra_init = virtio_crypto_ablkcipher_init,
	585	.cra_exit = virtio_crypto_ablkcipher_exit,
	586	.cra_u = {
	587	.ablkcipher = {
	588	.setkey = virtio_crypto_ablkcipher_setkey,
	589	.decrypt = virtio_crypto_ablkcipher_decrypt,
	590	.encrypt = virtio_crypto_ablkcipher_encrypt,
	591	.min_keysize = AES_MIN_KEY_SIZE,
	592	.max_keysize = AES_MAX_KEY_SIZE,
	593	.ivsize = AES_BLOCK_SIZE,
	594	},
	595	},
	596	} };
	597
	598	int virtio_crypto_algs_register(void)
	599	{
	600	int ret = 0;
	601
	602	mutex_lock(&algs_lock);
	603	if (++virtio_crypto_active_devs != 1)
	604	goto unlock;
	605
	606	ret = crypto_register_algs(virtio_crypto_algs,
	607	ARRAY_SIZE(virtio_crypto_algs));
	608	if (ret)
	609	virtio_crypto_active_devs--;
	610
	611	unlock:
	612	mutex_unlock(&algs_lock);
	613	return ret;
	614	}
	615
	616	void virtio_crypto_algs_unregister(void)
	617	{
	618	mutex_lock(&algs_lock);
	619	if (--virtio_crypto_active_devs != 0)
	620	goto unlock;
	621
	622	crypto_unregister_algs(virtio_crypto_algs,
	623	ARRAY_SIZE(virtio_crypto_algs));
	624
	625	unlock:
	626	mutex_unlock(&algs_lock);
	627	}