[linux-2.6-block.git] / drivers / crypto / rockchip / rk3288_crypto.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Crypto acceleration support for Rockchip RK3288
 *
 * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd
 *
 * Author: Zain Wang <zain.wang@rock-chips.com>
 *
 * Some ideas are from marvell-cesa.c and s5p-sss.c driver.
 */

#include "rk3288_crypto.h"
#include <crypto/engine.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/skcipher.h>
#include <linux/clk.h>
#include <linux/dma-mapping.h>
#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/io.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/of.h>
#include <linux/reset.h>
#include <linux/spinlock.h>

static struct rockchip_ip rocklist = {
	.dev_list = LIST_HEAD_INIT(rocklist.dev_list),
	.lock = __SPIN_LOCK_UNLOCKED(rocklist.lock),
};

struct rk_crypto_info *get_rk_crypto(void)
{
	struct rk_crypto_info *first;

	spin_lock(&rocklist.lock);
	first = list_first_entry_or_null(&rocklist.dev_list,
					 struct rk_crypto_info, list);
	list_rotate_left(&rocklist.dev_list);
	spin_unlock(&rocklist.lock);
	return first;
}

static const struct rk_variant rk3288_variant = {
	.num_clks = 4,
	.rkclks = {
		{ "sclk", 150000000},
	}
};

static const struct rk_variant rk3328_variant = {
	.num_clks = 3,
};

static const struct rk_variant rk3399_variant = {
	.num_clks = 3,
};

static int rk_crypto_get_clks(struct rk_crypto_info *dev)
{
	int i, j, err;
	unsigned long cr;

	dev->num_clks = devm_clk_bulk_get_all(dev->dev, &dev->clks);
	if (dev->num_clks < dev->variant->num_clks) {
		dev_err(dev->dev, "Missing clocks, got %d instead of %d\n",
			dev->num_clks, dev->variant->num_clks);
		return -EINVAL;
	}

	for (i = 0; i < dev->num_clks; i++) {
		cr = clk_get_rate(dev->clks[i].clk);
		for (j = 0; j < ARRAY_SIZE(dev->variant->rkclks); j++) {
			if (dev->variant->rkclks[j].max == 0)
				continue;
			if (strcmp(dev->variant->rkclks[j].name, dev->clks[i].id))
				continue;
			if (cr > dev->variant->rkclks[j].max) {
				err = clk_set_rate(dev->clks[i].clk,
						   dev->variant->rkclks[j].max);
				if (err)
					dev_err(dev->dev, "Fail downclocking %s from %lu to %lu\n",
						dev->variant->rkclks[j].name, cr,
						dev->variant->rkclks[j].max);
				else
					dev_info(dev->dev, "Downclocking %s from %lu to %lu\n",
						 dev->variant->rkclks[j].name, cr,
						 dev->variant->rkclks[j].max);
			}
		}
	}
	return 0;
}

static int rk_crypto_enable_clk(struct rk_crypto_info *dev)
{
	int err;

	err = clk_bulk_prepare_enable(dev->num_clks, dev->clks);
	if (err)
		dev_err(dev->dev, "Could not enable clock clks\n");

	return err;
}

static void rk_crypto_disable_clk(struct rk_crypto_info *dev)
{
	clk_bulk_disable_unprepare(dev->num_clks, dev->clks);
}

/*
 * Power management strategy: The device is suspended until a request
 * is handled. For avoiding suspend/resume yoyo, the autosuspend is set to 2s.
 */
static int rk_crypto_pm_suspend(struct device *dev)
{
	struct rk_crypto_info *rkdev = dev_get_drvdata(dev);

	rk_crypto_disable_clk(rkdev);
	reset_control_assert(rkdev->rst);

	return 0;
}

static int rk_crypto_pm_resume(struct device *dev)
{
	struct rk_crypto_info *rkdev = dev_get_drvdata(dev);
	int ret;

	ret = rk_crypto_enable_clk(rkdev);
	if (ret)
		return ret;

	reset_control_deassert(rkdev->rst);
	return 0;

}

static const struct dev_pm_ops rk_crypto_pm_ops = {
	SET_RUNTIME_PM_OPS(rk_crypto_pm_suspend, rk_crypto_pm_resume, NULL)
};

static int rk_crypto_pm_init(struct rk_crypto_info *rkdev)
{
	int err;

	pm_runtime_use_autosuspend(rkdev->dev);
	pm_runtime_set_autosuspend_delay(rkdev->dev, 2000);

	err = pm_runtime_set_suspended(rkdev->dev);
	if (err)
		return err;
	pm_runtime_enable(rkdev->dev);
	return err;
}

static void rk_crypto_pm_exit(struct rk_crypto_info *rkdev)
{
	pm_runtime_disable(rkdev->dev);
}

static irqreturn_t rk_crypto_irq_handle(int irq, void *dev_id)
{
	struct rk_crypto_info *dev  = platform_get_drvdata(dev_id);
	u32 interrupt_status;

	interrupt_status = CRYPTO_READ(dev, RK_CRYPTO_INTSTS);
	CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, interrupt_status);

	dev->status = 1;
	if (interrupt_status & 0x0a) {
		dev_warn(dev->dev, "DMA Error\n");
		dev->status = 0;
	}
	complete(&dev->complete);

	return IRQ_HANDLED;
}

static struct rk_crypto_tmp *rk_cipher_algs[] = {
	&rk_ecb_aes_alg,
	&rk_cbc_aes_alg,
	&rk_ecb_des_alg,
	&rk_cbc_des_alg,
	&rk_ecb_des3_ede_alg,
	&rk_cbc_des3_ede_alg,
	&rk_ahash_sha1,
	&rk_ahash_sha256,
	&rk_ahash_md5,
};

static int rk_crypto_debugfs_show(struct seq_file *seq, void *v)
{
	struct rk_crypto_info *dd;
	unsigned int i;

	spin_lock(&rocklist.lock);
	list_for_each_entry(dd, &rocklist.dev_list, list) {
		seq_printf(seq, "%s %s requests: %lu\n",
			   dev_driver_string(dd->dev), dev_name(dd->dev),
			   dd->nreq);
	}
	spin_unlock(&rocklist.lock);

	for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) {
		if (!rk_cipher_algs[i]->dev)
			continue;
		switch (rk_cipher_algs[i]->type) {
		case CRYPTO_ALG_TYPE_SKCIPHER:
			seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
				   rk_cipher_algs[i]->alg.skcipher.base.base.cra_driver_name,
				   rk_cipher_algs[i]->alg.skcipher.base.base.cra_name,
				   rk_cipher_algs[i]->stat_req, rk_cipher_algs[i]->stat_fb);
			seq_printf(seq, "\tfallback due to length: %lu\n",
				   rk_cipher_algs[i]->stat_fb_len);
			seq_printf(seq, "\tfallback due to alignment: %lu\n",
				   rk_cipher_algs[i]->stat_fb_align);
			seq_printf(seq, "\tfallback due to SGs: %lu\n",
				   rk_cipher_algs[i]->stat_fb_sgdiff);
			break;
		case CRYPTO_ALG_TYPE_AHASH:
			seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
				   rk_cipher_algs[i]->alg.hash.base.halg.base.cra_driver_name,
				   rk_cipher_algs[i]->alg.hash.base.halg.base.cra_name,
				   rk_cipher_algs[i]->stat_req, rk_cipher_algs[i]->stat_fb);
			break;
		}
	}
	return 0;
}

DEFINE_SHOW_ATTRIBUTE(rk_crypto_debugfs);

static void register_debugfs(struct rk_crypto_info *crypto_info)
{
	struct dentry *dbgfs_dir __maybe_unused;
	struct dentry *dbgfs_stats __maybe_unused;

	/* Ignore error of debugfs */
	dbgfs_dir = debugfs_create_dir("rk3288_crypto", NULL);
	dbgfs_stats = debugfs_create_file("stats", 0444, dbgfs_dir, &rocklist,
					  &rk_crypto_debugfs_fops);

#ifdef CONFIG_CRYPTO_DEV_ROCKCHIP_DEBUG
	rocklist.dbgfs_dir = dbgfs_dir;
	rocklist.dbgfs_stats = dbgfs_stats;
#endif
}

static int rk_crypto_register(struct rk_crypto_info *crypto_info)
{
	unsigned int i, k;
	int err = 0;

	for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) {
		rk_cipher_algs[i]->dev = crypto_info;
		switch (rk_cipher_algs[i]->type) {
		case CRYPTO_ALG_TYPE_SKCIPHER:
			dev_info(crypto_info->dev, "Register %s as %s\n",
				 rk_cipher_algs[i]->alg.skcipher.base.base.cra_name,
				 rk_cipher_algs[i]->alg.skcipher.base.base.cra_driver_name);
			err = crypto_engine_register_skcipher(&rk_cipher_algs[i]->alg.skcipher);
			break;
		case CRYPTO_ALG_TYPE_AHASH:
			dev_info(crypto_info->dev, "Register %s as %s\n",
				 rk_cipher_algs[i]->alg.hash.base.halg.base.cra_name,
				 rk_cipher_algs[i]->alg.hash.base.halg.base.cra_driver_name);
			err = crypto_engine_register_ahash(&rk_cipher_algs[i]->alg.hash);
			break;
		default:
			dev_err(crypto_info->dev, "unknown algorithm\n");
		}
		if (err)
			goto err_cipher_algs;
	}
	return 0;

err_cipher_algs:
	for (k = 0; k < i; k++) {
		if (rk_cipher_algs[i]->type == CRYPTO_ALG_TYPE_SKCIPHER)
			crypto_engine_unregister_skcipher(&rk_cipher_algs[k]->alg.skcipher);
		else
			crypto_engine_unregister_ahash(&rk_cipher_algs[i]->alg.hash);
	}
	return err;
}

static void rk_crypto_unregister(void)
{
	unsigned int i;

	for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) {
		if (rk_cipher_algs[i]->type == CRYPTO_ALG_TYPE_SKCIPHER)
			crypto_engine_unregister_skcipher(&rk_cipher_algs[i]->alg.skcipher);
		else
			crypto_engine_unregister_ahash(&rk_cipher_algs[i]->alg.hash);
	}
}

static const struct of_device_id crypto_of_id_table[] = {
	{ .compatible = "rockchip,rk3288-crypto",
	  .data = &rk3288_variant,
	},
	{ .compatible = "rockchip,rk3328-crypto",
	  .data = &rk3328_variant,
	},
	{ .compatible = "rockchip,rk3399-crypto",
	  .data = &rk3399_variant,
	},
	{}
};
MODULE_DEVICE_TABLE(of, crypto_of_id_table);

static int rk_crypto_probe(struct platform_device *pdev)
{
	struct device *dev = &pdev->dev;
	struct rk_crypto_info *crypto_info, *first;
	int err = 0;

	crypto_info = devm_kzalloc(&pdev->dev,
				   sizeof(*crypto_info), GFP_KERNEL);
	if (!crypto_info) {
		err = -ENOMEM;
		goto err_crypto;
	}

	crypto_info->dev = &pdev->dev;
	platform_set_drvdata(pdev, crypto_info);

	crypto_info->variant = of_device_get_match_data(&pdev->dev);
	if (!crypto_info->variant) {
		dev_err(&pdev->dev, "Missing variant\n");
		return -EINVAL;
	}

	crypto_info->rst = devm_reset_control_array_get_exclusive(dev);
	if (IS_ERR(crypto_info->rst)) {
		err = PTR_ERR(crypto_info->rst);
		goto err_crypto;
	}

	reset_control_assert(crypto_info->rst);
	usleep_range(10, 20);
	reset_control_deassert(crypto_info->rst);

	crypto_info->reg = devm_platform_ioremap_resource(pdev, 0);
	if (IS_ERR(crypto_info->reg)) {
		err = PTR_ERR(crypto_info->reg);
		goto err_crypto;
	}

	err = rk_crypto_get_clks(crypto_info);
	if (err)
		goto err_crypto;

	crypto_info->irq = platform_get_irq(pdev, 0);
	if (crypto_info->irq < 0) {
		err = crypto_info->irq;
		goto err_crypto;
	}

	err = devm_request_irq(&pdev->dev, crypto_info->irq,
			       rk_crypto_irq_handle, IRQF_SHARED,
			       "rk-crypto", pdev);

	if (err) {
		dev_err(&pdev->dev, "irq request failed.\n");
		goto err_crypto;
	}

	crypto_info->engine = crypto_engine_alloc_init(&pdev->dev, true);
	if (!crypto_info->engine) {
		err = -ENOMEM;
		goto err_crypto;
	}

	crypto_engine_start(crypto_info->engine);
	init_completion(&crypto_info->complete);

	err = rk_crypto_pm_init(crypto_info);
	if (err)
		goto err_pm;

	spin_lock(&rocklist.lock);
	first = list_first_entry_or_null(&rocklist.dev_list,
					 struct rk_crypto_info, list);
	list_add_tail(&crypto_info->list, &rocklist.dev_list);
	spin_unlock(&rocklist.lock);

	if (!first) {
		err = rk_crypto_register(crypto_info);
		if (err) {
			dev_err(dev, "Fail to register crypto algorithms");
			goto err_register_alg;
		}

		register_debugfs(crypto_info);
	}

	return 0;

err_register_alg:
	rk_crypto_pm_exit(crypto_info);
err_pm:
	crypto_engine_exit(crypto_info->engine);
err_crypto:
	dev_err(dev, "Crypto Accelerator not successfully registered\n");
	return err;
}

static void rk_crypto_remove(struct platform_device *pdev)
{
	struct rk_crypto_info *crypto_tmp = platform_get_drvdata(pdev);
	struct rk_crypto_info *first;

	spin_lock_bh(&rocklist.lock);
	list_del(&crypto_tmp->list);
	first = list_first_entry_or_null(&rocklist.dev_list,
					 struct rk_crypto_info, list);
	spin_unlock_bh(&rocklist.lock);

	if (!first) {
#ifdef CONFIG_CRYPTO_DEV_ROCKCHIP_DEBUG
		debugfs_remove_recursive(rocklist.dbgfs_dir);
#endif
		rk_crypto_unregister();
	}
	rk_crypto_pm_exit(crypto_tmp);
	crypto_engine_exit(crypto_tmp->engine);
}

static struct platform_driver crypto_driver = {
	.probe		= rk_crypto_probe,
	.remove_new	= rk_crypto_remove,
	.driver		= {
		.name	= "rk3288-crypto",
		.pm		= &rk_crypto_pm_ops,
		.of_match_table	= crypto_of_id_table,
	},
};

module_platform_driver(crypto_driver);

MODULE_AUTHOR("Zain Wang <zain.wang@rock-chips.com>");
MODULE_DESCRIPTION("Support for Rockchip's cryptographic engine");
MODULE_LICENSE("GPL");
Commit	Line	Data
75a6faf6	1	// SPDX-License-Identifier: GPL-2.0-only
433cd2c6 ZW	2	/*
	3	* Crypto acceleration support for Rockchip RK3288
	4	*
	5	* Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd
	6	*
	7	* Author: Zain Wang <zain.wang@rock-chips.com>
	8	*
433cd2c6 ZW	9	* Some ideas are from marvell-cesa.c and s5p-sss.c driver.
	10	*/
	11
	12	#include "rk3288_crypto.h"
1a15d26c HX	13	#include <crypto/engine.h>
	14	#include <crypto/internal/hash.h>
	15	#include <crypto/internal/skcipher.h>
	16	#include <linux/clk.h>
0c3dc787	17	#include <linux/dma-mapping.h>
1a15d26c HX	18	#include <linux/debugfs.h>
	19	#include <linux/delay.h>
	20	#include <linux/err.h>
	21	#include <linux/kernel.h>
	22	#include <linux/io.h>
433cd2c6 ZW	23	#include <linux/module.h>
	24	#include <linux/platform_device.h>
	25	#include <linux/of.h>
433cd2c6	26	#include <linux/reset.h>
1a15d26c	27	#include <linux/spinlock.h>
433cd2c6	28
9dcd71c8 CL	29	static struct rockchip_ip rocklist = {
	30	.dev_list = LIST_HEAD_INIT(rocklist.dev_list),
	31	.lock = __SPIN_LOCK_UNLOCKED(rocklist.lock),
	32	};
	33
	34	struct rk_crypto_info *get_rk_crypto(void)
	35	{
	36	struct rk_crypto_info *first;
	37
	38	spin_lock(&rocklist.lock);
	39	first = list_first_entry_or_null(&rocklist.dev_list,
	40	struct rk_crypto_info, list);
	41	list_rotate_left(&rocklist.dev_list);
	42	spin_unlock(&rocklist.lock);
	43	return first;
	44	}
	45
e220e671 CL	46	static const struct rk_variant rk3288_variant = {
	47	.num_clks = 4,
	48	.rkclks = {
	49	{ "sclk", 150000000},
	50	}
	51	};
	52
	53	static const struct rk_variant rk3328_variant = {
	54	.num_clks = 3,
	55	};
	56
9dcd71c8 CL	57	static const struct rk_variant rk3399_variant = {
	58	.num_clks = 3,
	59	};
	60
e220e671 CL	61	static int rk_crypto_get_clks(struct rk_crypto_info *dev)
	62	{
	63	int i, j, err;
	64	unsigned long cr;
	65
	66	dev->num_clks = devm_clk_bulk_get_all(dev->dev, &dev->clks);
	67	if (dev->num_clks < dev->variant->num_clks) {
	68	dev_err(dev->dev, "Missing clocks, got %d instead of %d\n",
	69	dev->num_clks, dev->variant->num_clks);
	70	return -EINVAL;
	71	}
	72
	73	for (i = 0; i < dev->num_clks; i++) {
	74	cr = clk_get_rate(dev->clks[i].clk);
	75	for (j = 0; j < ARRAY_SIZE(dev->variant->rkclks); j++) {
	76	if (dev->variant->rkclks[j].max == 0)
	77	continue;
	78	if (strcmp(dev->variant->rkclks[j].name, dev->clks[i].id))
	79	continue;
	80	if (cr > dev->variant->rkclks[j].max) {
	81	err = clk_set_rate(dev->clks[i].clk,
	82	dev->variant->rkclks[j].max);
	83	if (err)
	84	dev_err(dev->dev, "Fail downclocking %s from %lu to %lu\n",
	85	dev->variant->rkclks[j].name, cr,
	86	dev->variant->rkclks[j].max);
	87	else
	88	dev_info(dev->dev, "Downclocking %s from %lu to %lu\n",
	89	dev->variant->rkclks[j].name, cr,
	90	dev->variant->rkclks[j].max);
	91	}
	92	}
	93	}
	94	return 0;
	95	}
	96
433cd2c6 ZW	97	static int rk_crypto_enable_clk(struct rk_crypto_info *dev)
	98	{
	99	int err;
	100
3a6fd464 CL	101	err = clk_bulk_prepare_enable(dev->num_clks, dev->clks);
	102	if (err)
	103	dev_err(dev->dev, "Could not enable clock clks\n");
	104
433cd2c6 ZW	105	return err;
	106	}
	107
	108	static void rk_crypto_disable_clk(struct rk_crypto_info *dev)
	109	{
3a6fd464	110	clk_bulk_disable_unprepare(dev->num_clks, dev->clks);
433cd2c6 ZW	111	}
433cd2c6 ZW	112
a216be39	113	/*
9dcd71c8 CL	114	* Power management strategy: The device is suspended until a request
9dcd71c8 CL	115	* is handled. For avoiding suspend/resume yoyo, the autosuspend is set to 2s.
a216be39 CL	116	*/
	117	static int rk_crypto_pm_suspend(struct device *dev)
	118	{
	119	struct rk_crypto_info *rkdev = dev_get_drvdata(dev);
	120
	121	rk_crypto_disable_clk(rkdev);
6f611925 CL	122	reset_control_assert(rkdev->rst);
6f611925 CL	123
a216be39 CL	124	return 0;
	125	}
	126
	127	static int rk_crypto_pm_resume(struct device *dev)
	128	{
	129	struct rk_crypto_info *rkdev = dev_get_drvdata(dev);
6f611925 CL	130	int ret;
	131
	132	ret = rk_crypto_enable_clk(rkdev);
	133	if (ret)
	134	return ret;
	135
	136	reset_control_deassert(rkdev->rst);
	137	return 0;
a216be39	138
a216be39 CL	139	}
	140
	141	static const struct dev_pm_ops rk_crypto_pm_ops = {
	142	SET_RUNTIME_PM_OPS(rk_crypto_pm_suspend, rk_crypto_pm_resume, NULL)
	143	};
	144
	145	static int rk_crypto_pm_init(struct rk_crypto_info *rkdev)
	146	{
	147	int err;
	148
	149	pm_runtime_use_autosuspend(rkdev->dev);
	150	pm_runtime_set_autosuspend_delay(rkdev->dev, 2000);
	151
	152	err = pm_runtime_set_suspended(rkdev->dev);
	153	if (err)
	154	return err;
	155	pm_runtime_enable(rkdev->dev);
	156	return err;
	157	}
	158
	159	static void rk_crypto_pm_exit(struct rk_crypto_info *rkdev)
	160	{
	161	pm_runtime_disable(rkdev->dev);
	162	}
	163
433cd2c6 ZW	164	static irqreturn_t rk_crypto_irq_handle(int irq, void *dev_id)
	165	{
	166	struct rk_crypto_info *dev = platform_get_drvdata(dev_id);
	167	u32 interrupt_status;
433cd2c6	168
433cd2c6 ZW	169	interrupt_status = CRYPTO_READ(dev, RK_CRYPTO_INTSTS);
433cd2c6 ZW	170	CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, interrupt_status);
641eacd1	171
57d67c6e	172	dev->status = 1;
433cd2c6 ZW	173	if (interrupt_status & 0x0a) {
433cd2c6 ZW	174	dev_warn(dev->dev, "DMA Error\n");
57d67c6e	175	dev->status = 0;
433cd2c6	176	}
57d67c6e	177	complete(&dev->complete);
641eacd1	178
433cd2c6 ZW	179	return IRQ_HANDLED;
	180	}
	181
433cd2c6 ZW	182	static struct rk_crypto_tmp *rk_cipher_algs[] = {
	183	&rk_ecb_aes_alg,
	184	&rk_cbc_aes_alg,
	185	&rk_ecb_des_alg,
	186	&rk_cbc_des_alg,
	187	&rk_ecb_des3_ede_alg,
	188	&rk_cbc_des3_ede_alg,
bfd927ff ZW	189	&rk_ahash_sha1,
	190	&rk_ahash_sha256,
	191	&rk_ahash_md5,
433cd2c6 ZW	192	};
433cd2c6 ZW	193
48d904d4 CL	194	static int rk_crypto_debugfs_show(struct seq_file seq, void v)
48d904d4 CL	195	{
9dcd71c8	196	struct rk_crypto_info *dd;
48d904d4 CL	197	unsigned int i;
48d904d4 CL	198
9dcd71c8 CL	199	spin_lock(&rocklist.lock);
	200	list_for_each_entry(dd, &rocklist.dev_list, list) {
	201	seq_printf(seq, "%s %s requests: %lu\n",
	202	dev_driver_string(dd->dev), dev_name(dd->dev),
	203	dd->nreq);
	204	}
	205	spin_unlock(&rocklist.lock);
	206
48d904d4 CL	207	for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) {
	208	if (!rk_cipher_algs[i]->dev)
	209	continue;
	210	switch (rk_cipher_algs[i]->type) {
	211	case CRYPTO_ALG_TYPE_SKCIPHER:
	212	seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
1a15d26c HX	213	rk_cipher_algs[i]->alg.skcipher.base.base.cra_driver_name,
1a15d26c HX	214	rk_cipher_algs[i]->alg.skcipher.base.base.cra_name,
48d904d4 CL	215	rk_cipher_algs[i]->stat_req, rk_cipher_algs[i]->stat_fb);
	216	seq_printf(seq, "\tfallback due to length: %lu\n",
	217	rk_cipher_algs[i]->stat_fb_len);
	218	seq_printf(seq, "\tfallback due to alignment: %lu\n",
	219	rk_cipher_algs[i]->stat_fb_align);
	220	seq_printf(seq, "\tfallback due to SGs: %lu\n",
	221	rk_cipher_algs[i]->stat_fb_sgdiff);
	222	break;
	223	case CRYPTO_ALG_TYPE_AHASH:
	224	seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
1a15d26c HX	225	rk_cipher_algs[i]->alg.hash.base.halg.base.cra_driver_name,
1a15d26c HX	226	rk_cipher_algs[i]->alg.hash.base.halg.base.cra_name,
48d904d4 CL	227	rk_cipher_algs[i]->stat_req, rk_cipher_algs[i]->stat_fb);
	228	break;
	229	}
	230	}
	231	return 0;
	232	}
	233
	234	DEFINE_SHOW_ATTRIBUTE(rk_crypto_debugfs);
48d904d4	235
9dcd71c8 CL	236	static void register_debugfs(struct rk_crypto_info *crypto_info)
9dcd71c8 CL	237	{
1a15d26c HX	238	struct dentry *dbgfs_dir __maybe_unused;
	239	struct dentry *dbgfs_stats __maybe_unused;
	240
9dcd71c8	241	/* Ignore error of debugfs */
1a15d26c HX	242	dbgfs_dir = debugfs_create_dir("rk3288_crypto", NULL);
	243	dbgfs_stats = debugfs_create_file("stats", 0444, dbgfs_dir, &rocklist,
	244	&rk_crypto_debugfs_fops);
	245
	246	#ifdef CONFIG_CRYPTO_DEV_ROCKCHIP_DEBUG
	247	rocklist.dbgfs_dir = dbgfs_dir;
	248	rocklist.dbgfs_stats = dbgfs_stats;
9dcd71c8 CL	249	#endif
	250	}
	251
433cd2c6 ZW	252	static int rk_crypto_register(struct rk_crypto_info *crypto_info)
	253	{
	254	unsigned int i, k;
	255	int err = 0;
	256
	257	for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) {
	258	rk_cipher_algs[i]->dev = crypto_info;
6d55c4a2 CL	259	switch (rk_cipher_algs[i]->type) {
	260	case CRYPTO_ALG_TYPE_SKCIPHER:
	261	dev_info(crypto_info->dev, "Register %s as %s\n",
1a15d26c HX	262	rk_cipher_algs[i]->alg.skcipher.base.base.cra_name,
	263	rk_cipher_algs[i]->alg.skcipher.base.base.cra_driver_name);
	264	err = crypto_engine_register_skcipher(&rk_cipher_algs[i]->alg.skcipher);
6d55c4a2 CL	265	break;
	266	case CRYPTO_ALG_TYPE_AHASH:
	267	dev_info(crypto_info->dev, "Register %s as %s\n",
1a15d26c HX	268	rk_cipher_algs[i]->alg.hash.base.halg.base.cra_name,
	269	rk_cipher_algs[i]->alg.hash.base.halg.base.cra_driver_name);
	270	err = crypto_engine_register_ahash(&rk_cipher_algs[i]->alg.hash);
6d55c4a2 CL	271	break;
	272	default:
	273	dev_err(crypto_info->dev, "unknown algorithm\n");
	274	}
433cd2c6 ZW	275	if (err)
	276	goto err_cipher_algs;
	277	}
	278	return 0;
	279
	280	err_cipher_algs:
bfd927ff	281	for (k = 0; k < i; k++) {
6d55c4a2	282	if (rk_cipher_algs[i]->type == CRYPTO_ALG_TYPE_SKCIPHER)
1a15d26c	283	crypto_engine_unregister_skcipher(&rk_cipher_algs[k]->alg.skcipher);
bfd927ff	284	else
1a15d26c	285	crypto_engine_unregister_ahash(&rk_cipher_algs[i]->alg.hash);
bfd927ff	286	}
433cd2c6 ZW	287	return err;
	288	}
	289
	290	static void rk_crypto_unregister(void)
	291	{
	292	unsigned int i;
	293
bfd927ff	294	for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) {
6d55c4a2	295	if (rk_cipher_algs[i]->type == CRYPTO_ALG_TYPE_SKCIPHER)
1a15d26c	296	crypto_engine_unregister_skcipher(&rk_cipher_algs[i]->alg.skcipher);
bfd927ff	297	else
1a15d26c	298	crypto_engine_unregister_ahash(&rk_cipher_algs[i]->alg.hash);
bfd927ff	299	}
433cd2c6 ZW	300	}
433cd2c6 ZW	301
433cd2c6	302	static const struct of_device_id crypto_of_id_table[] = {
e220e671 CL	303	{ .compatible = "rockchip,rk3288-crypto",
	304	.data = &rk3288_variant,
	305	},
	306	{ .compatible = "rockchip,rk3328-crypto",
	307	.data = &rk3328_variant,
	308	},
9dcd71c8 CL	309	{ .compatible = "rockchip,rk3399-crypto",
	310	.data = &rk3399_variant,
	311	},
433cd2c6 ZW	312	{}
	313	};
	314	MODULE_DEVICE_TABLE(of, crypto_of_id_table);
	315
	316	static int rk_crypto_probe(struct platform_device *pdev)
	317	{
433cd2c6	318	struct device *dev = &pdev->dev;
9dcd71c8	319	struct rk_crypto_info crypto_info, first;
433cd2c6 ZW	320	int err = 0;
	321
	322	crypto_info = devm_kzalloc(&pdev->dev,
	323	sizeof(*crypto_info), GFP_KERNEL);
	324	if (!crypto_info) {
	325	err = -ENOMEM;
	326	goto err_crypto;
	327	}
	328
e220e671 CL	329	crypto_info->dev = &pdev->dev;
	330	platform_set_drvdata(pdev, crypto_info);
	331
	332	crypto_info->variant = of_device_get_match_data(&pdev->dev);
	333	if (!crypto_info->variant) {
	334	dev_err(&pdev->dev, "Missing variant\n");
	335	return -EINVAL;
	336	}
	337
c5a1e104	338	crypto_info->rst = devm_reset_control_array_get_exclusive(dev);
433cd2c6 ZW	339	if (IS_ERR(crypto_info->rst)) {
	340	err = PTR_ERR(crypto_info->rst);
	341	goto err_crypto;
	342	}
	343
	344	reset_control_assert(crypto_info->rst);
	345	usleep_range(10, 20);
	346	reset_control_deassert(crypto_info->rst);
	347
72174473	348	crypto_info->reg = devm_platform_ioremap_resource(pdev, 0);
433cd2c6 ZW	349	if (IS_ERR(crypto_info->reg)) {
	350	err = PTR_ERR(crypto_info->reg);
	351	goto err_crypto;
	352	}
	353
e220e671 CL	354	err = rk_crypto_get_clks(crypto_info);
e220e671 CL	355	if (err)
433cd2c6	356	goto err_crypto;
433cd2c6 ZW	357
	358	crypto_info->irq = platform_get_irq(pdev, 0);
	359	if (crypto_info->irq < 0) {
433cd2c6 ZW	360	err = crypto_info->irq;
	361	goto err_crypto;
	362	}
	363
	364	err = devm_request_irq(&pdev->dev, crypto_info->irq,
	365	rk_crypto_irq_handle, IRQF_SHARED,
	366	"rk-crypto", pdev);
	367
	368	if (err) {
8ccd9c8c	369	dev_err(&pdev->dev, "irq request failed.\n");
433cd2c6 ZW	370	goto err_crypto;
	371	}
	372
57d67c6e	373	crypto_info->engine = crypto_engine_alloc_init(&pdev->dev, true);
a24e3b58 KZ	374	if (!crypto_info->engine) {
	375	err = -ENOMEM;
	376	goto err_crypto;
	377	}
	378
57d67c6e CL	379	crypto_engine_start(crypto_info->engine);
57d67c6e CL	380	init_completion(&crypto_info->complete);
433cd2c6	381
a216be39 CL	382	err = rk_crypto_pm_init(crypto_info);
	383	if (err)
	384	goto err_pm;
433cd2c6	385
9dcd71c8 CL	386	spin_lock(&rocklist.lock);
	387	first = list_first_entry_or_null(&rocklist.dev_list,
	388	struct rk_crypto_info, list);
	389	list_add_tail(&crypto_info->list, &rocklist.dev_list);
	390	spin_unlock(&rocklist.lock);
	391
	392	if (!first) {
	393	err = rk_crypto_register(crypto_info);
	394	if (err) {
	395	dev_err(dev, "Fail to register crypto algorithms");
	396	goto err_register_alg;
	397	}
433cd2c6	398
9dcd71c8 CL	399	register_debugfs(crypto_info);
9dcd71c8 CL	400	}
48d904d4	401
433cd2c6 ZW	402	return 0;
	403
	404	err_register_alg:
a216be39 CL	405	rk_crypto_pm_exit(crypto_info);
a216be39 CL	406	err_pm:
57d67c6e	407	crypto_engine_exit(crypto_info->engine);
433cd2c6	408	err_crypto:
57d67c6e	409	dev_err(dev, "Crypto Accelerator not successfully registered\n");
433cd2c6 ZW	410	return err;
	411	}
	412
0a5cb261	413	static void rk_crypto_remove(struct platform_device *pdev)
433cd2c6 ZW	414	{
433cd2c6 ZW	415	struct rk_crypto_info *crypto_tmp = platform_get_drvdata(pdev);
9dcd71c8 CL	416	struct rk_crypto_info *first;
	417
	418	spin_lock_bh(&rocklist.lock);
	419	list_del(&crypto_tmp->list);
	420	first = list_first_entry_or_null(&rocklist.dev_list,
	421	struct rk_crypto_info, list);
	422	spin_unlock_bh(&rocklist.lock);
433cd2c6	423
9dcd71c8	424	if (!first) {
48d904d4	425	#ifdef CONFIG_CRYPTO_DEV_ROCKCHIP_DEBUG
9dcd71c8	426	debugfs_remove_recursive(rocklist.dbgfs_dir);
48d904d4	427	#endif
9dcd71c8 CL	428	rk_crypto_unregister();
9dcd71c8 CL	429	}
a216be39	430	rk_crypto_pm_exit(crypto_tmp);
57d67c6e	431	crypto_engine_exit(crypto_tmp->engine);
433cd2c6 ZW	432	}
	433
	434	static struct platform_driver crypto_driver = {
	435	.probe = rk_crypto_probe,
0a5cb261	436	.remove_new = rk_crypto_remove,
433cd2c6 ZW	437	.driver = {
433cd2c6 ZW	438	.name = "rk3288-crypto",
a216be39	439	.pm = &rk_crypto_pm_ops,
433cd2c6 ZW	440	.of_match_table = crypto_of_id_table,
	441	},
	442	};
	443
	444	module_platform_driver(crypto_driver);
	445
	446	MODULE_AUTHOR("Zain Wang <zain.wang@rock-chips.com>");
	447	MODULE_DESCRIPTION("Support for Rockchip's cryptographic engine");
	448	MODULE_LICENSE("GPL");