[linux-2.6-block.git] / drivers / crypto / qat / qat_dh895xcc / adf_drv.c

/*
  This file is provided under a dual BSD/GPLv2 license.  When using or
  redistributing this file, you may do so under either license.

  GPL LICENSE SUMMARY
  Copyright(c) 2014 Intel Corporation.
  This program is free software; you can redistribute it and/or modify
  it under the terms of version 2 of the GNU General Public License as
  published by the Free Software Foundation.

  This program is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  General Public License for more details.

  Contact Information:
  qat-linux@intel.com

  BSD LICENSE
  Copyright(c) 2014 Intel Corporation.
  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions
  are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the
      distribution.
    * Neither the name of Intel Corporation nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/platform_device.h>
#include <linux/workqueue.h>
#include <linux/io.h>
#include <adf_accel_devices.h>
#include <adf_common_drv.h>
#include <adf_cfg.h>
#include <adf_transport_access_macros.h>
#include "adf_dh895xcc_hw_data.h"
#include "adf_drv.h"

static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME;

#define ADF_SYSTEM_DEVICE(device_id) \
	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}

static const struct pci_device_id adf_pci_tbl[] = {
	ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
	{0,}
};
MODULE_DEVICE_TABLE(pci, adf_pci_tbl);

static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent);
static void adf_remove(struct pci_dev *dev);

static struct pci_driver adf_driver = {
	.id_table = adf_pci_tbl,
	.name = adf_driver_name,
	.probe = adf_probe,
	.remove = adf_remove
};

static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
{
	struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
	int i;

	adf_dev_shutdown(accel_dev);

	for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
		struct adf_bar *bar = &accel_pci_dev->pci_bars[i];

		if (bar->virt_addr)
			pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
	}

	if (accel_dev->hw_device) {
		switch (accel_dev->hw_device->pci_dev_id) {
		case ADF_DH895XCC_PCI_DEVICE_ID:
			adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
			break;
		default:
			break;
		}
		kfree(accel_dev->hw_device);
	}
	adf_cfg_dev_remove(accel_dev);
	debugfs_remove(accel_dev->debugfs_dir);
	adf_devmgr_rm_dev(accel_dev);
	pci_release_regions(accel_pci_dev->pci_dev);
	pci_disable_device(accel_pci_dev->pci_dev);
	kfree(accel_dev);
}

static int adf_dev_configure(struct adf_accel_dev *accel_dev)
{
	int cpus = num_online_cpus();
	int banks = GET_MAX_BANKS(accel_dev);
	int instances = min(cpus, banks);
	char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
	int i;
	unsigned long val;

	if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
		goto err;
	if (adf_cfg_section_add(accel_dev, "Accelerator0"))
		goto err;
	for (i = 0; i < instances; i++) {
		val = i;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
			 i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
		val = 128;
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 512;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 0;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 2;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 4;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 8;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 10;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = 12;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i);
		if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
						key, (void *)&val, ADF_DEC))
			goto err;

		val = ADF_COALESCING_DEF_TIME;
		snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
		if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
						key, (void *)&val, ADF_DEC))
			goto err;
	}

	val = i;
	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
					ADF_NUM_CY, (void *)&val, ADF_DEC))
		goto err;

	set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
	return 0;
err:
	dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
	return -EINVAL;
}

static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
	struct adf_accel_dev *accel_dev;
	struct adf_accel_pci *accel_pci_dev;
	struct adf_hw_device_data *hw_data;
	char name[ADF_DEVICE_NAME_LENGTH];
	unsigned int i, bar_nr;
	int ret;

	switch (ent->device) {
	case ADF_DH895XCC_PCI_DEVICE_ID:
		break;
	default:
		dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
		return -ENODEV;
	}

	if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
		/* If the accelerator is connected to a node with no memory
		 * there is no point in using the accelerator since the remote
		 * memory transaction will be very slow. */
		dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
		return -EINVAL;
	}

	accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
				 dev_to_node(&pdev->dev));
	if (!accel_dev)
		return -ENOMEM;

	INIT_LIST_HEAD(&accel_dev->crypto_list);

	/* Add accel device to accel table.
	 * This should be called before adf_cleanup_accel is called */
	if (adf_devmgr_add_dev(accel_dev)) {
		dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
		kfree(accel_dev);
		return -EFAULT;
	}

	accel_dev->owner = THIS_MODULE;
	/* Allocate and configure device configuration structure */
	hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL,
			       dev_to_node(&pdev->dev));
	if (!hw_data) {
		ret = -ENOMEM;
		goto out_err;
	}

	accel_dev->hw_device = hw_data;
	switch (ent->device) {
	case ADF_DH895XCC_PCI_DEVICE_ID:
		adf_init_hw_data_dh895xcc(accel_dev->hw_device);
		break;
	default:
		return -ENODEV;
	}
	accel_pci_dev = &accel_dev->accel_pci_dev;
	pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
	pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET,
			      &hw_data->fuses);

	/* Get Accelerators and Accelerators Engines masks */
	hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
	hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
	accel_pci_dev->sku = hw_data->get_sku(hw_data);
	accel_pci_dev->pci_dev = pdev;
	/* If the device has no acceleration engines then ignore it. */
	if (!hw_data->accel_mask || !hw_data->ae_mask ||
	    ((~hw_data->ae_mask) & 0x01)) {
		dev_err(&pdev->dev, "No acceleration units found");
		ret = -EFAULT;
		goto out_err;
	}

	/* Create dev top level debugfs entry */
	snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX,
		 hw_data->dev_class->name, hw_data->instance_id);
	accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
	if (!accel_dev->debugfs_dir) {
		dev_err(&pdev->dev, "Could not create debugfs dir\n");
		ret = -EINVAL;
		goto out_err;
	}

	/* Create device configuration table */
	ret = adf_cfg_dev_add(accel_dev);
	if (ret)
		goto out_err;

	pcie_set_readrq(pdev, 1024);

	/* enable PCI device */
	if (pci_enable_device(pdev)) {
		ret = -EFAULT;
		goto out_err;
	}

	/* set dma identifier */
	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
		if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
			dev_err(&pdev->dev, "No usable DMA configuration\n");
			ret = -EFAULT;
			goto out_err;
		} else {
			pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
		}

	} else {
		pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
	}

	if (pci_request_regions(pdev, adf_driver_name)) {
		ret = -EFAULT;
		goto out_err;
	}

	/* Read accelerator capabilities mask */
	pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET,
			      &hw_data->accel_capabilities_mask);

	/* Find and map all the device's BARS */
	for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
		struct adf_bar *bar = &accel_pci_dev->pci_bars[i];

		bar_nr = i * 2;
		bar->base_addr = pci_resource_start(pdev, bar_nr);
		if (!bar->base_addr)
			break;
		bar->size = pci_resource_len(pdev, bar_nr);
		bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
		if (!bar->virt_addr) {
			dev_err(&pdev->dev, "Failed to map BAR %d\n", i);
			ret = -EFAULT;
			goto out_err;
		}
	}
	pci_set_master(pdev);

	if (adf_enable_aer(accel_dev, &adf_driver)) {
		dev_err(&pdev->dev, "Failed to enable aer\n");
		ret = -EFAULT;
		goto out_err;
	}

	if (pci_save_state(pdev)) {
		dev_err(&pdev->dev, "Failed to save pci state\n");
		ret = -ENOMEM;
		goto out_err;
	}

	ret = adf_dev_configure(accel_dev);
	if (ret)
		goto out_err;

	ret = adf_dev_init(accel_dev);
	if (ret)
		goto out_err;

	ret = adf_dev_start(accel_dev);
	if (ret) {
		adf_dev_stop(accel_dev);
		goto out_err;
	}

	return 0;
out_err:
	adf_cleanup_accel(accel_dev);
	return ret;
}

static void adf_remove(struct pci_dev *pdev)
{
	struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);

	if (!accel_dev) {
		pr_err("QAT: Driver removal failed\n");
		return;
	}
	if (adf_dev_stop(accel_dev))
		dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");
	adf_disable_aer(accel_dev);
	adf_cleanup_accel(accel_dev);
}

static int __init adfdrv_init(void)
{
	request_module("intel_qat");
	if (qat_admin_register())
		return -EFAULT;

	if (pci_register_driver(&adf_driver)) {
		pr_err("QAT: Driver initialization failed\n");
		return -EFAULT;
	}
	return 0;
}

static void __exit adfdrv_release(void)
{
	pci_unregister_driver(&adf_driver);
	qat_admin_unregister();
}

module_init(adfdrv_init);
module_exit(adfdrv_release);

MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel");
MODULE_FIRMWARE(ADF_DH895XCC_FW);
MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
MODULE_VERSION(ADF_DRV_VERSION);
Commit	Line	Data
7afa232e TS	1	/*
	2	This file is provided under a dual BSD/GPLv2 license. When using or
	3	redistributing this file, you may do so under either license.
	4
	5	GPL LICENSE SUMMARY
	6	Copyright(c) 2014 Intel Corporation.
	7	This program is free software; you can redistribute it and/or modify
	8	it under the terms of version 2 of the GNU General Public License as
	9	published by the Free Software Foundation.
	10
	11	This program is distributed in the hope that it will be useful, but
	12	WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	General Public License for more details.
	15
	16	Contact Information:
	17	qat-linux@intel.com
	18
	19	BSD LICENSE
	20	Copyright(c) 2014 Intel Corporation.
	21	Redistribution and use in source and binary forms, with or without
	22	modification, are permitted provided that the following conditions
	23	are met:
	24
	25	* Redistributions of source code must retain the above copyright
	26	notice, this list of conditions and the following disclaimer.
	27	* Redistributions in binary form must reproduce the above copyright
	28	notice, this list of conditions and the following disclaimer in
	29	the documentation and/or other materials provided with the
	30	distribution.
	31	* Neither the name of Intel Corporation nor the names of its
	32	contributors may be used to endorse or promote products derived
	33	from this software without specific prior written permission.
	34
	35	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	36	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	37	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	38	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	39	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	40	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	41	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	42	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	43	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	44	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	45	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	46	*/
	47	#include <linux/kernel.h>
	48	#include <linux/module.h>
	49	#include <linux/pci.h>
	50	#include <linux/init.h>
	51	#include <linux/types.h>
	52	#include <linux/fs.h>
	53	#include <linux/slab.h>
	54	#include <linux/errno.h>
	55	#include <linux/device.h>
	56	#include <linux/dma-mapping.h>
	57	#include <linux/platform_device.h>
	58	#include <linux/workqueue.h>
	59	#include <linux/io.h>
	60	#include <adf_accel_devices.h>
	61	#include <adf_common_drv.h>
	62	#include <adf_cfg.h>
	63	#include <adf_transport_access_macros.h>
	64	#include "adf_dh895xcc_hw_data.h"
65	#include "adf_drv.h"
66
67	static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME;
68
69	#define ADF_SYSTEM_DEVICE(device_id) \
70	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
71
72	static const struct pci_device_id adf_pci_tbl[] = {
73	ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
74	{0,}
75	};
76	MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
77
78	static int adf_probe(struct pci_dev dev, const struct pci_device_id ent);
79	static void adf_remove(struct pci_dev *dev);
80
81	static struct pci_driver adf_driver = {
82	.id_table = adf_pci_tbl,
83	.name = adf_driver_name,
84	.probe = adf_probe,
85	.remove = adf_remove
86	};
87
88	static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
89	{
90	struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
91	int i;
92
22e4dda0	93	adf_dev_shutdown(accel_dev);
7afa232e TS	94
	95	for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
	96	struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
	97
	98	if (bar->virt_addr)
	99	pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
	100	}
	101
	102	if (accel_dev->hw_device) {
	103	switch (accel_dev->hw_device->pci_dev_id) {
	104	case ADF_DH895XCC_PCI_DEVICE_ID:
	105	adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
	106	break;
	107	default:
	108	break;
	109	}
	110	kfree(accel_dev->hw_device);
	111	}
	112	adf_cfg_dev_remove(accel_dev);
	113	debugfs_remove(accel_dev->debugfs_dir);
	114	adf_devmgr_rm_dev(accel_dev);
	115	pci_release_regions(accel_pci_dev->pci_dev);
	116	pci_disable_device(accel_pci_dev->pci_dev);
	117	kfree(accel_dev);
	118	}
	119
22e4dda0	120	static int adf_dev_configure(struct adf_accel_dev *accel_dev)
7afa232e TS	121	{
	122	int cpus = num_online_cpus();
	123	int banks = GET_MAX_BANKS(accel_dev);
	124	int instances = min(cpus, banks);
	125	char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
	126	int i;
	127	unsigned long val;
	128
	129	if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
	130	goto err;
	131	if (adf_cfg_section_add(accel_dev, "Accelerator0"))
	132	goto err;
	133	for (i = 0; i < instances; i++) {
	134	val = i;
	135	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i);
	136	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	137	key, (void *)&val, ADF_DEC))
	138	goto err;
	139
	140	snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
	141	i);
	142	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	143	key, (void *)&val, ADF_DEC))
	144	goto err;
	145
	146	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
	147	val = 128;
	148	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	149	key, (void *)&val, ADF_DEC))
	150	goto err;
	151
	152	val = 512;
	153	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
	154	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	155	key, (void *)&val, ADF_DEC))
	156	goto err;
	157
	158	val = 0;
	159	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
	160	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	161	key, (void *)&val, ADF_DEC))
	162	goto err;
	163
	164	val = 2;
	165	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
	166	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	167	key, (void *)&val, ADF_DEC))
	168	goto err;
	169
	170	val = 4;
	171	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i);
	172	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	173	key, (void *)&val, ADF_DEC))
	174	goto err;
	175
	176	val = 8;
	177	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
	178	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
	179	key, (void *)&val, ADF_DEC))
	180	goto err;
	181
	182	val = 10;
	183	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
	184	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
185	key, (void *)&val, ADF_DEC))
186	goto err;
187
188	val = 12;
189	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i);
190	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
191	key, (void *)&val, ADF_DEC))
192	goto err;
193
194	val = ADF_COALESCING_DEF_TIME;
195	snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
196	if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
197	key, (void *)&val, ADF_DEC))
198	goto err;
199	}
200
201	val = i;
202	if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
203	ADF_NUM_CY, (void *)&val, ADF_DEC))
204	goto err;
205
206	set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
22e4dda0	207	return 0;
7afa232e TS	208	err:
	209	dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
	210	return -EINVAL;
	211	}
	212
	213	static int adf_probe(struct pci_dev pdev, const struct pci_device_id ent)
	214	{
	215	struct adf_accel_dev *accel_dev;
	216	struct adf_accel_pci *accel_pci_dev;
	217	struct adf_hw_device_data *hw_data;
7afa232e TS	218	char name[ADF_DEVICE_NAME_LENGTH];
7afa232e TS	219	unsigned int i, bar_nr;
7afa232e TS	220	int ret;
	221
	222	switch (ent->device) {
	223	case ADF_DH895XCC_PCI_DEVICE_ID:
	224	break;
	225	default:
	226	dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
	227	return -ENODEV;
	228	}
	229
09adc878 TS	230	if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
	231	/* If the accelerator is connected to a node with no memory
	232	* there is no point in using the accelerator since the remote
	233	* memory transaction will be very slow. */
	234	dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
	235	return -EINVAL;
	236	}
	237
	238	accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
64a31be3	239	dev_to_node(&pdev->dev));
7afa232e TS	240	if (!accel_dev)
	241	return -ENOMEM;
	242
7afa232e TS	243	INIT_LIST_HEAD(&accel_dev->crypto_list);
	244
	245	/* Add accel device to accel table.
	246	* This should be called before adf_cleanup_accel is called */
	247	if (adf_devmgr_add_dev(accel_dev)) {
	248	dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
	249	kfree(accel_dev);
	250	return -EFAULT;
	251	}
	252
	253	accel_dev->owner = THIS_MODULE;
	254	/* Allocate and configure device configuration structure */
09adc878 TS	255	hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL,
09adc878 TS	256	dev_to_node(&pdev->dev));
7afa232e TS	257	if (!hw_data) {
	258	ret = -ENOMEM;
	259	goto out_err;
	260	}
	261
	262	accel_dev->hw_device = hw_data;
	263	switch (ent->device) {
	264	case ADF_DH895XCC_PCI_DEVICE_ID:
	265	adf_init_hw_data_dh895xcc(accel_dev->hw_device);
	266	break;
	267	default:
	268	return -ENODEV;
	269	}
	270	accel_pci_dev = &accel_dev->accel_pci_dev;
	271	pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
	272	pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET,
	273	&hw_data->fuses);
	274
	275	/* Get Accelerators and Accelerators Engines masks */
	276	hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
	277	hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
	278	accel_pci_dev->sku = hw_data->get_sku(hw_data);
	279	accel_pci_dev->pci_dev = pdev;
	280	/* If the device has no acceleration engines then ignore it. */
	281	if (!hw_data->accel_mask \|\| !hw_data->ae_mask \|\|
	282	((~hw_data->ae_mask) & 0x01)) {
	283	dev_err(&pdev->dev, "No acceleration units found");
	284	ret = -EFAULT;
	285	goto out_err;
	286	}
	287
	288	/* Create dev top level debugfs entry */
	289	snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX,
	290	hw_data->dev_class->name, hw_data->instance_id);
	291	accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
	292	if (!accel_dev->debugfs_dir) {
	293	dev_err(&pdev->dev, "Could not create debugfs dir\n");
	294	ret = -EINVAL;
	295	goto out_err;
	296	}
	297
	298	/* Create device configuration table */
	299	ret = adf_cfg_dev_add(accel_dev);
	300	if (ret)
	301	goto out_err;
	302
e4fa1460 TS	303	pcie_set_readrq(pdev, 1024);
e4fa1460 TS	304
7afa232e TS	305	/* enable PCI device */
	306	if (pci_enable_device(pdev)) {
	307	ret = -EFAULT;
	308	goto out_err;
	309	}
	310
	311	/* set dma identifier */
	312	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
	313	if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
	314	dev_err(&pdev->dev, "No usable DMA configuration\n");
	315	ret = -EFAULT;
	316	goto out_err;
	317	} else {
	318	pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
	319	}
	320
	321	} else {
	322	pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
	323	}
	324
	325	if (pci_request_regions(pdev, adf_driver_name)) {
	326	ret = -EFAULT;
	327	goto out_err;
	328	}
	329
	330	/* Read accelerator capabilities mask */
	331	pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET,
	332	&hw_data->accel_capabilities_mask);
	333
	334	/* Find and map all the device's BARS */
	335	for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
	336	struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
	337
	338	bar_nr = i * 2;
	339	bar->base_addr = pci_resource_start(pdev, bar_nr);
	340	if (!bar->base_addr)
	341	break;
	342	bar->size = pci_resource_len(pdev, bar_nr);
	343	bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
	344	if (!bar->virt_addr) {
	345	dev_err(&pdev->dev, "Failed to map BAR %d\n", i);
	346	ret = -EFAULT;
	347	goto out_err;
	348	}
7afa232e TS	349	}
	350	pci_set_master(pdev);
	351
	352	if (adf_enable_aer(accel_dev, &adf_driver)) {
	353	dev_err(&pdev->dev, "Failed to enable aer\n");
	354	ret = -EFAULT;
	355	goto out_err;
	356	}
	357
7afa232e TS	358	if (pci_save_state(pdev)) {
	359	dev_err(&pdev->dev, "Failed to save pci state\n");
	360	ret = -ENOMEM;
	361	goto out_err;
	362	}
	363
22e4dda0 AB	364	ret = adf_dev_configure(accel_dev);
	365	if (ret)
	366	goto out_err;
	367
	368	ret = adf_dev_init(accel_dev);
	369	if (ret)
	370	goto out_err;
7afa232e	371
22e4dda0	372	ret = adf_dev_start(accel_dev);
7afa232e TS	373	if (ret) {
	374	adf_dev_stop(accel_dev);
	375	goto out_err;
	376	}
	377
	378	return 0;
	379	out_err:
	380	adf_cleanup_accel(accel_dev);
	381	return ret;
	382	}
	383
83ce01d2	384	static void adf_remove(struct pci_dev *pdev)
7afa232e TS	385	{
	386	struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
	387
	388	if (!accel_dev) {
	389	pr_err("QAT: Driver removal failed\n");
	390	return;
	391	}
	392	if (adf_dev_stop(accel_dev))
	393	dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");
	394	adf_disable_aer(accel_dev);
	395	adf_cleanup_accel(accel_dev);
	396	}
	397
	398	static int __init adfdrv_init(void)
	399	{
	400	request_module("intel_qat");
	401	if (qat_admin_register())
	402	return -EFAULT;
	403
	404	if (pci_register_driver(&adf_driver)) {
	405	pr_err("QAT: Driver initialization failed\n");
	406	return -EFAULT;
	407	}
	408	return 0;
	409	}
	410
	411	static void __exit adfdrv_release(void)
	412	{
	413	pci_unregister_driver(&adf_driver);
	414	qat_admin_unregister();
	415	}
	416
	417	module_init(adfdrv_init);
	418	module_exit(adfdrv_release);
	419
	420	MODULE_LICENSE("Dual BSD/GPL");
	421	MODULE_AUTHOR("Intel");
dcd93e83	422	MODULE_FIRMWARE(ADF_DH895XCC_FW);
7afa232e	423	MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
551d7ed2	424	MODULE_VERSION(ADF_DRV_VERSION);