[linux-2.6-block.git] / drivers / crypto / ccree / cc_driver.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */

#include <linux/kernel.h>
#include <linux/module.h>

#include <linux/crypto.h>
#include <linux/moduleparam.h>
#include <linux/types.h>
#include <linux/interrupt.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/of.h>
#include <linux/clk.h>
#include <linux/of_address.h>

#include "cc_driver.h"
#include "cc_request_mgr.h"
#include "cc_buffer_mgr.h"
#include "cc_debugfs.h"
#include "cc_cipher.h"
#include "cc_aead.h"
#include "cc_hash.h"
#include "cc_ivgen.h"
#include "cc_sram_mgr.h"
#include "cc_pm.h"
#include "cc_fips.h"

bool cc_dump_desc;
module_param_named(dump_desc, cc_dump_desc, bool, 0600);
MODULE_PARM_DESC(cc_dump_desc, "Dump descriptors to kernel log as debugging aid");

bool cc_dump_bytes;
module_param_named(dump_bytes, cc_dump_bytes, bool, 0600);
MODULE_PARM_DESC(cc_dump_bytes, "Dump buffers to kernel log as debugging aid");

struct cc_hw_data {
	char *name;
	enum cc_hw_rev rev;
	u32 sig;
};

/* Hardware revisions defs. */

static const struct cc_hw_data cc712_hw = {
	.name = "712", .rev = CC_HW_REV_712, .sig =  0xDCC71200U
};

static const struct cc_hw_data cc710_hw = {
	.name = "710", .rev = CC_HW_REV_710, .sig =  0xDCC63200U
};

static const struct cc_hw_data cc630p_hw = {
	.name = "630P", .rev = CC_HW_REV_630, .sig = 0xDCC63000U
};

static const struct of_device_id arm_ccree_dev_of_match[] = {
	{ .compatible = "arm,cryptocell-712-ree", .data = &cc712_hw },
	{ .compatible = "arm,cryptocell-710-ree", .data = &cc710_hw },
	{ .compatible = "arm,cryptocell-630p-ree", .data = &cc630p_hw },
	{}
};
MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match);

void __dump_byte_array(const char *name, const u8 *buf, size_t len)
{
	char prefix[64];

	if (!buf)
		return;

	snprintf(prefix, sizeof(prefix), "%s[%zu]: ", name, len);

	print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_ADDRESS, 16, 1, buf,
		       len, false);
}

static irqreturn_t cc_isr(int irq, void *dev_id)
{
	struct cc_drvdata *drvdata = (struct cc_drvdata *)dev_id;
	struct device *dev = drvdata_to_dev(drvdata);
	u32 irr;
	u32 imr;

	/* STAT_OP_TYPE_GENERIC STAT_PHASE_0: Interrupt */

	/* read the interrupt status */
	irr = cc_ioread(drvdata, CC_REG(HOST_IRR));
	dev_dbg(dev, "Got IRR=0x%08X\n", irr);
	if (irr == 0) { /* Probably shared interrupt line */
		dev_err(dev, "Got interrupt with empty IRR\n");
		return IRQ_NONE;
	}
	imr = cc_ioread(drvdata, CC_REG(HOST_IMR));

	/* clear interrupt - must be before processing events */
	cc_iowrite(drvdata, CC_REG(HOST_ICR), irr);

	drvdata->irq = irr;
	/* Completion interrupt - most probable */
	if (irr & CC_COMP_IRQ_MASK) {
		/* Mask AXI completion interrupt - will be unmasked in
		 * Deferred service handler
		 */
		cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | CC_COMP_IRQ_MASK);
		irr &= ~CC_COMP_IRQ_MASK;
		complete_request(drvdata);
	}
#ifdef CONFIG_CRYPTO_FIPS
	/* TEE FIPS interrupt */
	if (irr & CC_GPR0_IRQ_MASK) {
		/* Mask interrupt - will be unmasked in Deferred service
		 * handler
		 */
		cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | CC_GPR0_IRQ_MASK);
		irr &= ~CC_GPR0_IRQ_MASK;
		fips_handler(drvdata);
	}
#endif
	/* AXI error interrupt */
	if (irr & CC_AXI_ERR_IRQ_MASK) {
		u32 axi_err;

		/* Read the AXI error ID */
		axi_err = cc_ioread(drvdata, CC_REG(AXIM_MON_ERR));
		dev_dbg(dev, "AXI completion error: axim_mon_err=0x%08X\n",
			axi_err);

		irr &= ~CC_AXI_ERR_IRQ_MASK;
	}

	if (irr) {
		dev_dbg(dev, "IRR includes unknown cause bits (0x%08X)\n",
			irr);
		/* Just warning */
	}

	return IRQ_HANDLED;
}

int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe)
{
	unsigned int val, cache_params;
	struct device *dev = drvdata_to_dev(drvdata);

	/* Unmask all AXI interrupt sources AXI_CFG1 register */
	val = cc_ioread(drvdata, CC_REG(AXIM_CFG));
	cc_iowrite(drvdata, CC_REG(AXIM_CFG), val & ~CC_AXI_IRQ_MASK);
	dev_dbg(dev, "AXIM_CFG=0x%08X\n",
		cc_ioread(drvdata, CC_REG(AXIM_CFG)));

	/* Clear all pending interrupts */
	val = cc_ioread(drvdata, CC_REG(HOST_IRR));
	dev_dbg(dev, "IRR=0x%08X\n", val);
	cc_iowrite(drvdata, CC_REG(HOST_ICR), val);

	/* Unmask relevant interrupt cause */
	val = CC_COMP_IRQ_MASK | CC_AXI_ERR_IRQ_MASK;

	if (drvdata->hw_rev >= CC_HW_REV_712)
		val |= CC_GPR0_IRQ_MASK;

	cc_iowrite(drvdata, CC_REG(HOST_IMR), ~val);

	cache_params = (drvdata->coherent ? CC_COHERENT_CACHE_PARAMS : 0x0);

	val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));

	if (is_probe)
		dev_dbg(dev, "Cache params previous: 0x%08X\n", val);

	cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), cache_params);
	val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));

	if (is_probe)
		dev_dbg(dev, "Cache params current: 0x%08X (expect: 0x%08X)\n",
			val, cache_params);

	return 0;
}

static int init_cc_resources(struct platform_device *plat_dev)
{
	struct resource *req_mem_cc_regs = NULL;
	struct cc_drvdata *new_drvdata;
	struct device *dev = &plat_dev->dev;
	struct device_node *np = dev->of_node;
	u32 signature_val;
	u64 dma_mask;
	const struct cc_hw_data *hw_rev;
	const struct of_device_id *dev_id;
	struct clk *clk;
	int rc = 0;

	new_drvdata = devm_kzalloc(dev, sizeof(*new_drvdata), GFP_KERNEL);
	if (!new_drvdata)
		return -ENOMEM;

	dev_id = of_match_node(arm_ccree_dev_of_match, np);
	if (!dev_id)
		return -ENODEV;

	hw_rev = (struct cc_hw_data *)dev_id->data;
	new_drvdata->hw_rev_name = hw_rev->name;
	new_drvdata->hw_rev = hw_rev->rev;

	if (hw_rev->rev >= CC_HW_REV_712) {
		new_drvdata->hash_len_sz = HASH_LEN_SIZE_712;
		new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP);
		new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_712);
		new_drvdata->ver_offset = CC_REG(HOST_VERSION_712);
	} else {
		new_drvdata->hash_len_sz = HASH_LEN_SIZE_630;
		new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP8);
		new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_630);
		new_drvdata->ver_offset = CC_REG(HOST_VERSION_630);
	}

	platform_set_drvdata(plat_dev, new_drvdata);
	new_drvdata->plat_dev = plat_dev;

	clk = devm_clk_get(dev, NULL);
	if (IS_ERR(clk))
		switch (PTR_ERR(clk)) {
		/* Clock is optional so this might be fine */
		case -ENOENT:
			break;

		/* Clock not available, let's try again soon */
		case -EPROBE_DEFER:
			return -EPROBE_DEFER;

		default:
			dev_err(dev, "Error getting clock: %ld\n",
				PTR_ERR(clk));
			return PTR_ERR(clk);
		}
	new_drvdata->clk = clk;

	new_drvdata->coherent = of_dma_is_coherent(np);

	/* Get device resources */
	/* First CC registers space */
	req_mem_cc_regs = platform_get_resource(plat_dev, IORESOURCE_MEM, 0);
	/* Map registers space */
	new_drvdata->cc_base = devm_ioremap_resource(dev, req_mem_cc_regs);
	if (IS_ERR(new_drvdata->cc_base)) {
		dev_err(dev, "Failed to ioremap registers");
		return PTR_ERR(new_drvdata->cc_base);
	}

	dev_dbg(dev, "Got MEM resource (%s): %pR\n", req_mem_cc_regs->name,
		req_mem_cc_regs);
	dev_dbg(dev, "CC registers mapped from %pa to 0x%p\n",
		&req_mem_cc_regs->start, new_drvdata->cc_base);

	/* Then IRQ */
	new_drvdata->irq = platform_get_irq(plat_dev, 0);
	if (new_drvdata->irq < 0) {
		dev_err(dev, "Failed getting IRQ resource\n");
		return new_drvdata->irq;
	}

	rc = devm_request_irq(dev, new_drvdata->irq, cc_isr,
			      IRQF_SHARED, "ccree", new_drvdata);
	if (rc) {
		dev_err(dev, "Could not register to interrupt %d\n",
			new_drvdata->irq);
		return rc;
	}
	dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq);

	init_completion(&new_drvdata->hw_queue_avail);

	if (!plat_dev->dev.dma_mask)
		plat_dev->dev.dma_mask = &plat_dev->dev.coherent_dma_mask;

	dma_mask = DMA_BIT_MASK(DMA_BIT_MASK_LEN);
	while (dma_mask > 0x7fffffffUL) {
		if (dma_supported(&plat_dev->dev, dma_mask)) {
			rc = dma_set_coherent_mask(&plat_dev->dev, dma_mask);
			if (!rc)
				break;
		}
		dma_mask >>= 1;
	}

	if (rc) {
		dev_err(dev, "Failed in dma_set_mask, mask=%llx\n", dma_mask);
		return rc;
	}

	rc = cc_clk_on(new_drvdata);
	if (rc) {
		dev_err(dev, "Failed to enable clock");
		return rc;
	}

	/* Verify correct mapping */
	signature_val = cc_ioread(new_drvdata, new_drvdata->sig_offset);
	if (signature_val != hw_rev->sig) {
		dev_err(dev, "Invalid CC signature: SIGNATURE=0x%08X != expected=0x%08X\n",
			signature_val, hw_rev->sig);
		rc = -EINVAL;
		goto post_clk_err;
	}
	dev_dbg(dev, "CC SIGNATURE=0x%08X\n", signature_val);

	/* Display HW versions */
	dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X, Driver version %s\n",
		 hw_rev->name, cc_ioread(new_drvdata, new_drvdata->ver_offset),
		 DRV_MODULE_VERSION);

	rc = init_cc_regs(new_drvdata, true);
	if (rc) {
		dev_err(dev, "init_cc_regs failed\n");
		goto post_clk_err;
	}

	rc = cc_debugfs_init(new_drvdata);
	if (rc) {
		dev_err(dev, "Failed registering debugfs interface\n");
		goto post_regs_err;
	}

	rc = cc_fips_init(new_drvdata);
	if (rc) {
		dev_err(dev, "CC_FIPS_INIT failed 0x%x\n", rc);
		goto post_debugfs_err;
	}
	rc = cc_sram_mgr_init(new_drvdata);
	if (rc) {
		dev_err(dev, "cc_sram_mgr_init failed\n");
		goto post_fips_init_err;
	}

	new_drvdata->mlli_sram_addr =
		cc_sram_alloc(new_drvdata, MAX_MLLI_BUFF_SIZE);
	if (new_drvdata->mlli_sram_addr == NULL_SRAM_ADDR) {
		dev_err(dev, "Failed to alloc MLLI Sram buffer\n");
		rc = -ENOMEM;
		goto post_sram_mgr_err;
	}

	rc = cc_req_mgr_init(new_drvdata);
	if (rc) {
		dev_err(dev, "cc_req_mgr_init failed\n");
		goto post_sram_mgr_err;
	}

	rc = cc_buffer_mgr_init(new_drvdata);
	if (rc) {
		dev_err(dev, "buffer_mgr_init failed\n");
		goto post_req_mgr_err;
	}

	rc = cc_pm_init(new_drvdata);
	if (rc) {
		dev_err(dev, "ssi_power_mgr_init failed\n");
		goto post_buf_mgr_err;
	}

	rc = cc_ivgen_init(new_drvdata);
	if (rc) {
		dev_err(dev, "cc_ivgen_init failed\n");
		goto post_power_mgr_err;
	}

	/* Allocate crypto algs */
	rc = cc_cipher_alloc(new_drvdata);
	if (rc) {
		dev_err(dev, "cc_cipher_alloc failed\n");
		goto post_ivgen_err;
	}

	/* hash must be allocated before aead since hash exports APIs */
	rc = cc_hash_alloc(new_drvdata);
	if (rc) {
		dev_err(dev, "cc_hash_alloc failed\n");
		goto post_cipher_err;
	}

	rc = cc_aead_alloc(new_drvdata);
	if (rc) {
		dev_err(dev, "cc_aead_alloc failed\n");
		goto post_hash_err;
	}

	/* If we got here and FIPS mode is enabled
	 * it means all FIPS test passed, so let TEE
	 * know we're good.
	 */
	cc_set_ree_fips_status(new_drvdata, true);

	return 0;

post_hash_err:
	cc_hash_free(new_drvdata);
post_cipher_err:
	cc_cipher_free(new_drvdata);
post_ivgen_err:
	cc_ivgen_fini(new_drvdata);
post_power_mgr_err:
	cc_pm_fini(new_drvdata);
post_buf_mgr_err:
	 cc_buffer_mgr_fini(new_drvdata);
post_req_mgr_err:
	cc_req_mgr_fini(new_drvdata);
post_sram_mgr_err:
	cc_sram_mgr_fini(new_drvdata);
post_fips_init_err:
	cc_fips_fini(new_drvdata);
post_debugfs_err:
	cc_debugfs_fini(new_drvdata);
post_regs_err:
	fini_cc_regs(new_drvdata);
post_clk_err:
	cc_clk_off(new_drvdata);
	return rc;
}

void fini_cc_regs(struct cc_drvdata *drvdata)
{
	/* Mask all interrupts */
	cc_iowrite(drvdata, CC_REG(HOST_IMR), 0xFFFFFFFF);
}

static void cleanup_cc_resources(struct platform_device *plat_dev)
{
	struct cc_drvdata *drvdata =
		(struct cc_drvdata *)platform_get_drvdata(plat_dev);

	cc_aead_free(drvdata);
	cc_hash_free(drvdata);
	cc_cipher_free(drvdata);
	cc_ivgen_fini(drvdata);
	cc_pm_fini(drvdata);
	cc_buffer_mgr_fini(drvdata);
	cc_req_mgr_fini(drvdata);
	cc_sram_mgr_fini(drvdata);
	cc_fips_fini(drvdata);
	cc_debugfs_fini(drvdata);
	fini_cc_regs(drvdata);
	cc_clk_off(drvdata);
}

int cc_clk_on(struct cc_drvdata *drvdata)
{
	struct clk *clk = drvdata->clk;
	int rc;

	if (IS_ERR(clk))
		/* Not all devices have a clock associated with CCREE  */
		return 0;

	rc = clk_prepare_enable(clk);
	if (rc)
		return rc;

	return 0;
}

void cc_clk_off(struct cc_drvdata *drvdata)
{
	struct clk *clk = drvdata->clk;

	if (IS_ERR(clk))
		/* Not all devices have a clock associated with CCREE */
		return;

	clk_disable_unprepare(clk);
}

static int ccree_probe(struct platform_device *plat_dev)
{
	int rc;
	struct device *dev = &plat_dev->dev;

	/* Map registers space */
	rc = init_cc_resources(plat_dev);
	if (rc)
		return rc;

	dev_info(dev, "ARM ccree device initialized\n");

	return 0;
}

static int ccree_remove(struct platform_device *plat_dev)
{
	struct device *dev = &plat_dev->dev;

	dev_dbg(dev, "Releasing ccree resources...\n");

	cleanup_cc_resources(plat_dev);

	dev_info(dev, "ARM ccree device terminated\n");

	return 0;
}

static struct platform_driver ccree_driver = {
	.driver = {
		   .name = "ccree",
		   .of_match_table = arm_ccree_dev_of_match,
#ifdef CONFIG_PM
		   .pm = &ccree_pm,
#endif
	},
	.probe = ccree_probe,
	.remove = ccree_remove,
};

static int __init ccree_init(void)
{
	int ret;

	cc_hash_global_init();

	ret = cc_debugfs_global_init();
	if (ret)
		return ret;

	return platform_driver_register(&ccree_driver);
}
module_init(ccree_init);

static void __exit ccree_exit(void)
{
	platform_driver_unregister(&ccree_driver);
	cc_debugfs_global_fini();
}
module_exit(ccree_exit);

/* Module description */
MODULE_DESCRIPTION("ARM TrustZone CryptoCell REE Driver");
MODULE_VERSION(DRV_MODULE_VERSION);
MODULE_AUTHOR("ARM");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
4c3f9727 GBY	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
	3
	4	#include <linux/kernel.h>
	5	#include <linux/module.h>
	6
	7	#include <linux/crypto.h>
	8	#include <linux/moduleparam.h>
	9	#include <linux/types.h>
	10	#include <linux/interrupt.h>
	11	#include <linux/platform_device.h>
	12	#include <linux/slab.h>
	13	#include <linux/spinlock.h>
	14	#include <linux/of.h>
	15	#include <linux/clk.h>
	16	#include <linux/of_address.h>
	17
	18	#include "cc_driver.h"
	19	#include "cc_request_mgr.h"
	20	#include "cc_buffer_mgr.h"
	21	#include "cc_debugfs.h"
63ee04c8	22	#include "cc_cipher.h"
ff27e85a	23	#include "cc_aead.h"
63893811	24	#include "cc_hash.h"
4c3f9727 GBY	25	#include "cc_ivgen.h"
	26	#include "cc_sram_mgr.h"
	27	#include "cc_pm.h"
ab8ec965	28	#include "cc_fips.h"
4c3f9727 GBY	29
	30	bool cc_dump_desc;
	31	module_param_named(dump_desc, cc_dump_desc, bool, 0600);
	32	MODULE_PARM_DESC(cc_dump_desc, "Dump descriptors to kernel log as debugging aid");
	33
	34	bool cc_dump_bytes;
	35	module_param_named(dump_bytes, cc_dump_bytes, bool, 0600);
	36	MODULE_PARM_DESC(cc_dump_bytes, "Dump buffers to kernel log as debugging aid");
	37
27b3b22d GBY	38	struct cc_hw_data {
	39	char *name;
	40	enum cc_hw_rev rev;
	41	u32 sig;
	42	};
	43
	44	/* Hardware revisions defs. */
	45
	46	static const struct cc_hw_data cc712_hw = {
	47	.name = "712", .rev = CC_HW_REV_712, .sig = 0xDCC71200U
	48	};
	49
	50	static const struct cc_hw_data cc710_hw = {
	51	.name = "710", .rev = CC_HW_REV_710, .sig = 0xDCC63200U
	52	};
	53
	54	static const struct cc_hw_data cc630p_hw = {
	55	.name = "630P", .rev = CC_HW_REV_630, .sig = 0xDCC63000U
	56	};
	57
	58	static const struct of_device_id arm_ccree_dev_of_match[] = {
	59	{ .compatible = "arm,cryptocell-712-ree", .data = &cc712_hw },
	60	{ .compatible = "arm,cryptocell-710-ree", .data = &cc710_hw },
	61	{ .compatible = "arm,cryptocell-630p-ree", .data = &cc630p_hw },
	62	{}
	63	};
	64	MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match);
	65
4c3f9727 GBY	66	void __dump_byte_array(const char name, const u8 buf, size_t len)
	67	{
	68	char prefix[64];
	69
	70	if (!buf)
	71	return;
	72
	73	snprintf(prefix, sizeof(prefix), "%s[%zu]: ", name, len);
	74
	75	print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_ADDRESS, 16, 1, buf,
	76	len, false);
	77	}
	78
	79	static irqreturn_t cc_isr(int irq, void *dev_id)
	80	{
	81	struct cc_drvdata drvdata = (struct cc_drvdata )dev_id;
	82	struct device *dev = drvdata_to_dev(drvdata);
	83	u32 irr;
	84	u32 imr;
	85
	86	/* STAT_OP_TYPE_GENERIC STAT_PHASE_0: Interrupt */
	87
	88	/* read the interrupt status */
	89	irr = cc_ioread(drvdata, CC_REG(HOST_IRR));
	90	dev_dbg(dev, "Got IRR=0x%08X\n", irr);
	91	if (irr == 0) { /* Probably shared interrupt line */
	92	dev_err(dev, "Got interrupt with empty IRR\n");
	93	return IRQ_NONE;
	94	}
	95	imr = cc_ioread(drvdata, CC_REG(HOST_IMR));
	96
	97	/* clear interrupt - must be before processing events */
	98	cc_iowrite(drvdata, CC_REG(HOST_ICR), irr);
	99
	100	drvdata->irq = irr;
	101	/* Completion interrupt - most probable */
	102	if (irr & CC_COMP_IRQ_MASK) {
	103	/* Mask AXI completion interrupt - will be unmasked in
	104	* Deferred service handler
	105	*/
	106	cc_iowrite(drvdata, CC_REG(HOST_IMR), imr \| CC_COMP_IRQ_MASK);
	107	irr &= ~CC_COMP_IRQ_MASK;
	108	complete_request(drvdata);
	109	}
ab8ec965 GBY	110	#ifdef CONFIG_CRYPTO_FIPS
	111	/* TEE FIPS interrupt */
	112	if (irr & CC_GPR0_IRQ_MASK) {
	113	/* Mask interrupt - will be unmasked in Deferred service
	114	* handler
	115	*/
	116	cc_iowrite(drvdata, CC_REG(HOST_IMR), imr \| CC_GPR0_IRQ_MASK);
	117	irr &= ~CC_GPR0_IRQ_MASK;
	118	fips_handler(drvdata);
	119	}
	120	#endif
4c3f9727 GBY	121	/* AXI error interrupt */
	122	if (irr & CC_AXI_ERR_IRQ_MASK) {
	123	u32 axi_err;
	124
	125	/* Read the AXI error ID */
	126	axi_err = cc_ioread(drvdata, CC_REG(AXIM_MON_ERR));
	127	dev_dbg(dev, "AXI completion error: axim_mon_err=0x%08X\n",
	128	axi_err);
	129
	130	irr &= ~CC_AXI_ERR_IRQ_MASK;
	131	}
	132
	133	if (irr) {
	134	dev_dbg(dev, "IRR includes unknown cause bits (0x%08X)\n",
	135	irr);
	136	/* Just warning */
	137	}
	138
	139	return IRQ_HANDLED;
	140	}
	141
	142	int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe)
	143	{
	144	unsigned int val, cache_params;
	145	struct device *dev = drvdata_to_dev(drvdata);
	146
	147	/* Unmask all AXI interrupt sources AXI_CFG1 register */
	148	val = cc_ioread(drvdata, CC_REG(AXIM_CFG));
	149	cc_iowrite(drvdata, CC_REG(AXIM_CFG), val & ~CC_AXI_IRQ_MASK);
	150	dev_dbg(dev, "AXIM_CFG=0x%08X\n",
	151	cc_ioread(drvdata, CC_REG(AXIM_CFG)));
	152
	153	/* Clear all pending interrupts */
	154	val = cc_ioread(drvdata, CC_REG(HOST_IRR));
	155	dev_dbg(dev, "IRR=0x%08X\n", val);
	156	cc_iowrite(drvdata, CC_REG(HOST_ICR), val);
	157
	158	/* Unmask relevant interrupt cause */
27b3b22d GBY	159	val = CC_COMP_IRQ_MASK \| CC_AXI_ERR_IRQ_MASK;
	160
	161	if (drvdata->hw_rev >= CC_HW_REV_712)
	162	val \|= CC_GPR0_IRQ_MASK;
	163
	164	cc_iowrite(drvdata, CC_REG(HOST_IMR), ~val);
4c3f9727 GBY	165
	166	cache_params = (drvdata->coherent ? CC_COHERENT_CACHE_PARAMS : 0x0);
	167
	168	val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));
	169
	170	if (is_probe)
565018b8	171	dev_dbg(dev, "Cache params previous: 0x%08X\n", val);
4c3f9727 GBY	172
	173	cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), cache_params);
	174	val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));
	175
	176	if (is_probe)
565018b8 GBY	177	dev_dbg(dev, "Cache params current: 0x%08X (expect: 0x%08X)\n",
565018b8 GBY	178	val, cache_params);
4c3f9727 GBY	179
	180	return 0;
	181	}
	182
	183	static int init_cc_resources(struct platform_device *plat_dev)
	184	{
	185	struct resource *req_mem_cc_regs = NULL;
	186	struct cc_drvdata *new_drvdata;
	187	struct device *dev = &plat_dev->dev;
	188	struct device_node *np = dev->of_node;
	189	u32 signature_val;
	190	u64 dma_mask;
27b3b22d GBY	191	const struct cc_hw_data *hw_rev;
27b3b22d GBY	192	const struct of_device_id *dev_id;
35f859fc	193	struct clk *clk;
4c3f9727 GBY	194	int rc = 0;
	195
	196	new_drvdata = devm_kzalloc(dev, sizeof(*new_drvdata), GFP_KERNEL);
	197	if (!new_drvdata)
	198	return -ENOMEM;
	199
27b3b22d GBY	200	dev_id = of_match_node(arm_ccree_dev_of_match, np);
	201	if (!dev_id)
	202	return -ENODEV;
	203
	204	hw_rev = (struct cc_hw_data *)dev_id->data;
	205	new_drvdata->hw_rev_name = hw_rev->name;
	206	new_drvdata->hw_rev = hw_rev->rev;
	207
	208	if (hw_rev->rev >= CC_HW_REV_712) {
	209	new_drvdata->hash_len_sz = HASH_LEN_SIZE_712;
	210	new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP);
281a58c8 GBY	211	new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_712);
281a58c8 GBY	212	new_drvdata->ver_offset = CC_REG(HOST_VERSION_712);
27b3b22d GBY	213	} else {
	214	new_drvdata->hash_len_sz = HASH_LEN_SIZE_630;
	215	new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP8);
281a58c8 GBY	216	new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_630);
281a58c8 GBY	217	new_drvdata->ver_offset = CC_REG(HOST_VERSION_630);
27b3b22d GBY	218	}
27b3b22d GBY	219
4c3f9727 GBY	220	platform_set_drvdata(plat_dev, new_drvdata);
	221	new_drvdata->plat_dev = plat_dev;
	222
35f859fc GBY	223	clk = devm_clk_get(dev, NULL);
	224	if (IS_ERR(clk))
	225	switch (PTR_ERR(clk)) {
	226	/* Clock is optional so this might be fine */
	227	case -ENOENT:
	228	break;
	229
	230	/* Clock not available, let's try again soon */
	231	case -EPROBE_DEFER:
	232	return -EPROBE_DEFER;
	233
	234	default:
	235	dev_err(dev, "Error getting clock: %ld\n",
	236	PTR_ERR(clk));
	237	return PTR_ERR(clk);
	238	}
	239	new_drvdata->clk = clk;
	240
4c3f9727 GBY	241	new_drvdata->coherent = of_dma_is_coherent(np);
	242
	243	/* Get device resources */
	244	/* First CC registers space */
	245	req_mem_cc_regs = platform_get_resource(plat_dev, IORESOURCE_MEM, 0);
	246	/* Map registers space */
	247	new_drvdata->cc_base = devm_ioremap_resource(dev, req_mem_cc_regs);
	248	if (IS_ERR(new_drvdata->cc_base)) {
	249	dev_err(dev, "Failed to ioremap registers");
	250	return PTR_ERR(new_drvdata->cc_base);
	251	}
	252
	253	dev_dbg(dev, "Got MEM resource (%s): %pR\n", req_mem_cc_regs->name,
	254	req_mem_cc_regs);
	255	dev_dbg(dev, "CC registers mapped from %pa to 0x%p\n",
	256	&req_mem_cc_regs->start, new_drvdata->cc_base);
	257
	258	/* Then IRQ */
	259	new_drvdata->irq = platform_get_irq(plat_dev, 0);
	260	if (new_drvdata->irq < 0) {
	261	dev_err(dev, "Failed getting IRQ resource\n");
	262	return new_drvdata->irq;
	263	}
	264
	265	rc = devm_request_irq(dev, new_drvdata->irq, cc_isr,
	266	IRQF_SHARED, "ccree", new_drvdata);
	267	if (rc) {
	268	dev_err(dev, "Could not register to interrupt %d\n",
	269	new_drvdata->irq);
	270	return rc;
	271	}
	272	dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq);
	273
	274	init_completion(&new_drvdata->hw_queue_avail);
	275
	276	if (!plat_dev->dev.dma_mask)
	277	plat_dev->dev.dma_mask = &plat_dev->dev.coherent_dma_mask;
	278
	279	dma_mask = DMA_BIT_MASK(DMA_BIT_MASK_LEN);
	280	while (dma_mask > 0x7fffffffUL) {
	281	if (dma_supported(&plat_dev->dev, dma_mask)) {
	282	rc = dma_set_coherent_mask(&plat_dev->dev, dma_mask);
	283	if (!rc)
	284	break;
	285	}
	286	dma_mask >>= 1;
	287	}
	288
	289	if (rc) {
5e7b516a	290	dev_err(dev, "Failed in dma_set_mask, mask=%llx\n", dma_mask);
4c3f9727 GBY	291	return rc;
	292	}
	293
	294	rc = cc_clk_on(new_drvdata);
	295	if (rc) {
	296	dev_err(dev, "Failed to enable clock");
	297	return rc;
	298	}
	299
	300	/* Verify correct mapping */
281a58c8	301	signature_val = cc_ioread(new_drvdata, new_drvdata->sig_offset);
27b3b22d	302	if (signature_val != hw_rev->sig) {
4c3f9727	303	dev_err(dev, "Invalid CC signature: SIGNATURE=0x%08X != expected=0x%08X\n",
27b3b22d	304	signature_val, hw_rev->sig);
4c3f9727 GBY	305	rc = -EINVAL;
	306	goto post_clk_err;
	307	}
	308	dev_dbg(dev, "CC SIGNATURE=0x%08X\n", signature_val);
	309
	310	/* Display HW versions */
	311	dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X, Driver version %s\n",
281a58c8	312	hw_rev->name, cc_ioread(new_drvdata, new_drvdata->ver_offset),
4c3f9727 GBY	313	DRV_MODULE_VERSION);
	314
	315	rc = init_cc_regs(new_drvdata, true);
	316	if (rc) {
	317	dev_err(dev, "init_cc_regs failed\n");
	318	goto post_clk_err;
	319	}
	320
	321	rc = cc_debugfs_init(new_drvdata);
	322	if (rc) {
	323	dev_err(dev, "Failed registering debugfs interface\n");
	324	goto post_regs_err;
	325	}
	326
ab8ec965 GBY	327	rc = cc_fips_init(new_drvdata);
	328	if (rc) {
	329	dev_err(dev, "CC_FIPS_INIT failed 0x%x\n", rc);
	330	goto post_debugfs_err;
	331	}
4c3f9727 GBY	332	rc = cc_sram_mgr_init(new_drvdata);
	333	if (rc) {
	334	dev_err(dev, "cc_sram_mgr_init failed\n");
ab8ec965	335	goto post_fips_init_err;
4c3f9727 GBY	336	}
	337
	338	new_drvdata->mlli_sram_addr =
	339	cc_sram_alloc(new_drvdata, MAX_MLLI_BUFF_SIZE);
	340	if (new_drvdata->mlli_sram_addr == NULL_SRAM_ADDR) {
	341	dev_err(dev, "Failed to alloc MLLI Sram buffer\n");
	342	rc = -ENOMEM;
	343	goto post_sram_mgr_err;
	344	}
	345
	346	rc = cc_req_mgr_init(new_drvdata);
	347	if (rc) {
	348	dev_err(dev, "cc_req_mgr_init failed\n");
	349	goto post_sram_mgr_err;
	350	}
	351
	352	rc = cc_buffer_mgr_init(new_drvdata);
	353	if (rc) {
	354	dev_err(dev, "buffer_mgr_init failed\n");
	355	goto post_req_mgr_err;
	356	}
	357
	358	rc = cc_pm_init(new_drvdata);
	359	if (rc) {
	360	dev_err(dev, "ssi_power_mgr_init failed\n");
	361	goto post_buf_mgr_err;
	362	}
	363
	364	rc = cc_ivgen_init(new_drvdata);
	365	if (rc) {
	366	dev_err(dev, "cc_ivgen_init failed\n");
	367	goto post_power_mgr_err;
	368	}
	369
63ee04c8 GBY	370	/* Allocate crypto algs */
	371	rc = cc_cipher_alloc(new_drvdata);
	372	if (rc) {
	373	dev_err(dev, "cc_cipher_alloc failed\n");
	374	goto post_ivgen_err;
	375	}
	376
63893811 GBY	377	/* hash must be allocated before aead since hash exports APIs */
	378	rc = cc_hash_alloc(new_drvdata);
	379	if (rc) {
	380	dev_err(dev, "cc_hash_alloc failed\n");
	381	goto post_cipher_err;
	382	}
	383
ff27e85a GBY	384	rc = cc_aead_alloc(new_drvdata);
	385	if (rc) {
	386	dev_err(dev, "cc_aead_alloc failed\n");
	387	goto post_hash_err;
	388	}
	389
ab8ec965 GBY	390	/* If we got here and FIPS mode is enabled
	391	* it means all FIPS test passed, so let TEE
	392	* know we're good.
	393	*/
	394	cc_set_ree_fips_status(new_drvdata, true);
	395
4c3f9727 GBY	396	return 0;
4c3f9727 GBY	397
ff27e85a GBY	398	post_hash_err:
ff27e85a GBY	399	cc_hash_free(new_drvdata);
63893811 GBY	400	post_cipher_err:
63893811 GBY	401	cc_cipher_free(new_drvdata);
63ee04c8 GBY	402	post_ivgen_err:
63ee04c8 GBY	403	cc_ivgen_fini(new_drvdata);
4c3f9727 GBY	404	post_power_mgr_err:
	405	cc_pm_fini(new_drvdata);
	406	post_buf_mgr_err:
	407	cc_buffer_mgr_fini(new_drvdata);
	408	post_req_mgr_err:
	409	cc_req_mgr_fini(new_drvdata);
	410	post_sram_mgr_err:
	411	cc_sram_mgr_fini(new_drvdata);
ab8ec965 GBY	412	post_fips_init_err:
ab8ec965 GBY	413	cc_fips_fini(new_drvdata);
4c3f9727 GBY	414	post_debugfs_err:
	415	cc_debugfs_fini(new_drvdata);
	416	post_regs_err:
	417	fini_cc_regs(new_drvdata);
	418	post_clk_err:
	419	cc_clk_off(new_drvdata);
	420	return rc;
	421	}
	422
	423	void fini_cc_regs(struct cc_drvdata *drvdata)
	424	{
	425	/* Mask all interrupts */
	426	cc_iowrite(drvdata, CC_REG(HOST_IMR), 0xFFFFFFFF);
	427	}
	428
	429	static void cleanup_cc_resources(struct platform_device *plat_dev)
	430	{
	431	struct cc_drvdata *drvdata =
	432	(struct cc_drvdata *)platform_get_drvdata(plat_dev);
	433
ff27e85a	434	cc_aead_free(drvdata);
63893811	435	cc_hash_free(drvdata);
63ee04c8	436	cc_cipher_free(drvdata);
4c3f9727 GBY	437	cc_ivgen_fini(drvdata);
	438	cc_pm_fini(drvdata);
	439	cc_buffer_mgr_fini(drvdata);
	440	cc_req_mgr_fini(drvdata);
	441	cc_sram_mgr_fini(drvdata);
ab8ec965	442	cc_fips_fini(drvdata);
4c3f9727 GBY	443	cc_debugfs_fini(drvdata);
	444	fini_cc_regs(drvdata);
	445	cc_clk_off(drvdata);
	446	}
	447
	448	int cc_clk_on(struct cc_drvdata *drvdata)
	449	{
	450	struct clk *clk = drvdata->clk;
	451	int rc;
	452
	453	if (IS_ERR(clk))
	454	/* Not all devices have a clock associated with CCREE */
	455	return 0;
	456
	457	rc = clk_prepare_enable(clk);
	458	if (rc)
	459	return rc;
	460
	461	return 0;
	462	}
	463
	464	void cc_clk_off(struct cc_drvdata *drvdata)
	465	{
	466	struct clk *clk = drvdata->clk;
	467
	468	if (IS_ERR(clk))
	469	/* Not all devices have a clock associated with CCREE */
	470	return;
	471
	472	clk_disable_unprepare(clk);
	473	}
	474
	475	static int ccree_probe(struct platform_device *plat_dev)
	476	{
	477	int rc;
	478	struct device *dev = &plat_dev->dev;
	479
	480	/* Map registers space */
	481	rc = init_cc_resources(plat_dev);
	482	if (rc)
	483	return rc;
	484
	485	dev_info(dev, "ARM ccree device initialized\n");
	486
	487	return 0;
	488	}
	489
	490	static int ccree_remove(struct platform_device *plat_dev)
	491	{
	492	struct device *dev = &plat_dev->dev;
	493
	494	dev_dbg(dev, "Releasing ccree resources...\n");
	495
	496	cleanup_cc_resources(plat_dev);
	497
	498	dev_info(dev, "ARM ccree device terminated\n");
	499
	500	return 0;
	501	}
	502
4c3f9727 GBY	503	static struct platform_driver ccree_driver = {
	504	.driver = {
	505	.name = "ccree",
	506	.of_match_table = arm_ccree_dev_of_match,
	507	#ifdef CONFIG_PM
	508	.pm = &ccree_pm,
	509	#endif
	510	},
	511	.probe = ccree_probe,
	512	.remove = ccree_remove,
	513	};
	514
	515	static int __init ccree_init(void)
	516	{
	517	int ret;
	518
63893811 GBY	519	cc_hash_global_init();
63893811 GBY	520
4c3f9727 GBY	521	ret = cc_debugfs_global_init();
	522	if (ret)
	523	return ret;
	524
	525	return platform_driver_register(&ccree_driver);
	526	}
	527	module_init(ccree_init);
	528
	529	static void __exit ccree_exit(void)
	530	{
	531	platform_driver_unregister(&ccree_driver);
	532	cc_debugfs_global_fini();
	533	}
	534	module_exit(ccree_exit);
	535
	536	/* Module description */
	537	MODULE_DESCRIPTION("ARM TrustZone CryptoCell REE Driver");
	538	MODULE_VERSION(DRV_MODULE_VERSION);
	539	MODULE_AUTHOR("ARM");
	540	MODULE_LICENSE("GPL v2");