[linux-2.6-block.git] / drivers / vfio / pci / vfio_pci.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved
 *
 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
 *     Author: Alex Williamson <alex.williamson@redhat.com>
 *
 * Derived from original vfio:
 * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
 * Author: Tom Lyon, pugs@cisco.com
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/device.h>
#include <linux/eventfd.h>
#include <linux/file.h>
#include <linux/interrupt.h>
#include <linux/iommu.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/notifier.h>
#include <linux/pm_runtime.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/uaccess.h>

#include "vfio_pci_priv.h"

#define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
#define DRIVER_DESC     "VFIO PCI - User Level meta-driver"

static char ids[1024] __initdata;
module_param_string(ids, ids, sizeof(ids), 0);
MODULE_PARM_DESC(ids, "Initial PCI IDs to add to the vfio driver, format is \"vendor:device[:subvendor[:subdevice[:class[:class_mask]]]]\" and multiple comma separated entries can be specified");

static bool nointxmask;
module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(nointxmask,
		  "Disable support for PCI 2.3 style INTx masking.  If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag.");

#ifdef CONFIG_VFIO_PCI_VGA
static bool disable_vga;
module_param(disable_vga, bool, S_IRUGO);
MODULE_PARM_DESC(disable_vga, "Disable VGA resource access through vfio-pci");
#endif

static bool disable_idle_d3;
module_param(disable_idle_d3, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(disable_idle_d3,
		 "Disable using the PCI D3 low power state for idle, unused devices");

static bool enable_sriov;
#ifdef CONFIG_PCI_IOV
module_param(enable_sriov, bool, 0644);
MODULE_PARM_DESC(enable_sriov, "Enable support for SR-IOV configuration.  Enabling SR-IOV on a PF typically requires support of the userspace PF driver, enabling VFs without such support may result in non-functional VFs or PF.");
#endif

static bool disable_denylist;
module_param(disable_denylist, bool, 0444);
MODULE_PARM_DESC(disable_denylist, "Disable use of device denylist. Disabling the denylist allows binding to devices with known errata that may lead to exploitable stability or security issues when accessed by untrusted users.");

static bool vfio_pci_dev_in_denylist(struct pci_dev *pdev)
{
	switch (pdev->vendor) {
	case PCI_VENDOR_ID_INTEL:
		switch (pdev->device) {
		case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
		case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF:
		case PCI_DEVICE_ID_INTEL_QAT_C62X:
		case PCI_DEVICE_ID_INTEL_QAT_C62X_VF:
		case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
		case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF:
		case PCI_DEVICE_ID_INTEL_DSA_SPR0:
		case PCI_DEVICE_ID_INTEL_IAX_SPR0:
			return true;
		default:
			return false;
		}
	}

	return false;
}

static bool vfio_pci_is_denylisted(struct pci_dev *pdev)
{
	if (!vfio_pci_dev_in_denylist(pdev))
		return false;

	if (disable_denylist) {
		pci_warn(pdev,
			 "device denylist disabled - allowing device %04x:%04x.\n",
			 pdev->vendor, pdev->device);
		return false;
	}

	pci_warn(pdev, "%04x:%04x exists in vfio-pci device denylist, driver probing disallowed.\n",
		 pdev->vendor, pdev->device);

	return true;
}

static int vfio_pci_open_device(struct vfio_device *core_vdev)
{
	struct vfio_pci_core_device *vdev =
		container_of(core_vdev, struct vfio_pci_core_device, vdev);
	struct pci_dev *pdev = vdev->pdev;
	int ret;

	ret = vfio_pci_core_enable(vdev);
	if (ret)
		return ret;

	if (vfio_pci_is_intel_display(pdev)) {
		ret = vfio_pci_igd_init(vdev);
		if (ret && ret != -ENODEV) {
			pci_warn(pdev, "Failed to setup Intel IGD regions\n");
			vfio_pci_core_disable(vdev);
			return ret;
		}
	}

	vfio_pci_core_finish_enable(vdev);

	return 0;
}

static const struct vfio_device_ops vfio_pci_ops = {
	.name		= "vfio-pci",
	.init		= vfio_pci_core_init_dev,
	.release	= vfio_pci_core_release_dev,
	.open_device	= vfio_pci_open_device,
	.close_device	= vfio_pci_core_close_device,
	.ioctl		= vfio_pci_core_ioctl,
	.device_feature = vfio_pci_core_ioctl_feature,
	.read		= vfio_pci_core_read,
	.write		= vfio_pci_core_write,
	.mmap		= vfio_pci_core_mmap,
	.request	= vfio_pci_core_request,
	.match		= vfio_pci_core_match,
	.bind_iommufd	= vfio_iommufd_physical_bind,
	.unbind_iommufd	= vfio_iommufd_physical_unbind,
	.attach_ioas	= vfio_iommufd_physical_attach_ioas,
	.detach_ioas	= vfio_iommufd_physical_detach_ioas,
	.pasid_attach_ioas	= vfio_iommufd_physical_pasid_attach_ioas,
	.pasid_detach_ioas	= vfio_iommufd_physical_pasid_detach_ioas,
};

static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
	struct vfio_pci_core_device *vdev;
	int ret;

	if (vfio_pci_is_denylisted(pdev))
		return -EINVAL;

	vdev = vfio_alloc_device(vfio_pci_core_device, vdev, &pdev->dev,
				 &vfio_pci_ops);
	if (IS_ERR(vdev))
		return PTR_ERR(vdev);

	dev_set_drvdata(&pdev->dev, vdev);
	ret = vfio_pci_core_register_device(vdev);
	if (ret)
		goto out_put_vdev;
	return 0;

out_put_vdev:
	vfio_put_device(&vdev->vdev);
	return ret;
}

static void vfio_pci_remove(struct pci_dev *pdev)
{
	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);

	vfio_pci_core_unregister_device(vdev);
	vfio_put_device(&vdev->vdev);
}

static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
{
	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);

	if (!enable_sriov)
		return -ENOENT;

	return vfio_pci_core_sriov_configure(vdev, nr_virtfn);
}

static const struct pci_device_id vfio_pci_table[] = {
	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_ANY_ID, PCI_ANY_ID) }, /* match all by default */
	{}
};

MODULE_DEVICE_TABLE(pci, vfio_pci_table);

static struct pci_driver vfio_pci_driver = {
	.name			= "vfio-pci",
	.id_table		= vfio_pci_table,
	.probe			= vfio_pci_probe,
	.remove			= vfio_pci_remove,
	.sriov_configure	= vfio_pci_sriov_configure,
	.err_handler		= &vfio_pci_core_err_handlers,
	.driver_managed_dma	= true,
};

static void __init vfio_pci_fill_ids(void)
{
	char *p, *id;
	int rc;

	/* no ids passed actually */
	if (ids[0] == '\0')
		return;

	/* add ids specified in the module parameter */
	p = ids;
	while ((id = strsep(&p, ","))) {
		unsigned int vendor, device, subvendor = PCI_ANY_ID,
			subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
		int fields;

		if (!strlen(id))
			continue;

		fields = sscanf(id, "%x:%x:%x:%x:%x:%x",
				&vendor, &device, &subvendor, &subdevice,
				&class, &class_mask);

		if (fields < 2) {
			pr_warn("invalid id string \"%s\"\n", id);
			continue;
		}

		rc = pci_add_dynid(&vfio_pci_driver, vendor, device,
				   subvendor, subdevice, class, class_mask, 0);
		if (rc)
			pr_warn("failed to add dynamic id [%04x:%04x[%04x:%04x]] class %#08x/%08x (%d)\n",
				vendor, device, subvendor, subdevice,
				class, class_mask, rc);
		else
			pr_info("add [%04x:%04x[%04x:%04x]] class %#08x/%08x\n",
				vendor, device, subvendor, subdevice,
				class, class_mask);
	}
}

static int __init vfio_pci_init(void)
{
	int ret;
	bool is_disable_vga = true;

#ifdef CONFIG_VFIO_PCI_VGA
	is_disable_vga = disable_vga;
#endif

	vfio_pci_core_set_params(nointxmask, is_disable_vga, disable_idle_d3);

	/* Register and scan for devices */
	ret = pci_register_driver(&vfio_pci_driver);
	if (ret)
		return ret;

	vfio_pci_fill_ids();

	if (disable_denylist)
		pr_warn("device denylist disabled.\n");

	return 0;
}
module_init(vfio_pci_init);

static void __exit vfio_pci_cleanup(void)
{
	pci_unregister_driver(&vfio_pci_driver);
}
module_exit(vfio_pci_cleanup);

MODULE_LICENSE("GPL v2");
MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);
Commit	Line	Data
ff53edf6 MG	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved
	4	*
	5	* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
	6	* Author: Alex Williamson <alex.williamson@redhat.com>
	7	*
	8	* Derived from original vfio:
	9	* Copyright 2010 Cisco Systems, Inc. All rights reserved.
	10	* Author: Tom Lyon, pugs@cisco.com
	11	*/
	12
	13	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	14
	15	#include <linux/device.h>
	16	#include <linux/eventfd.h>
	17	#include <linux/file.h>
	18	#include <linux/interrupt.h>
	19	#include <linux/iommu.h>
	20	#include <linux/module.h>
	21	#include <linux/mutex.h>
	22	#include <linux/notifier.h>
	23	#include <linux/pm_runtime.h>
	24	#include <linux/slab.h>
	25	#include <linux/types.h>
	26	#include <linux/uaccess.h>
	27
e34a0425	28	#include "vfio_pci_priv.h"
ff53edf6 MG	29
	30	#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
	31	#define DRIVER_DESC "VFIO PCI - User Level meta-driver"
	32
	33	static char ids[1024] __initdata;
	34	module_param_string(ids, ids, sizeof(ids), 0);
	35	MODULE_PARM_DESC(ids, "Initial PCI IDs to add to the vfio driver, format is \"vendor:device[:subvendor[:subdevice[:class[:class_mask]]]]\" and multiple comma separated entries can be specified");
	36
c61302aa YH	37	static bool nointxmask;
	38	module_param_named(nointxmask, nointxmask, bool, S_IRUGO \| S_IWUSR);
	39	MODULE_PARM_DESC(nointxmask,
	40	"Disable support for PCI 2.3 style INTx masking. If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag.");
	41
	42	#ifdef CONFIG_VFIO_PCI_VGA
	43	static bool disable_vga;
	44	module_param(disable_vga, bool, S_IRUGO);
	45	MODULE_PARM_DESC(disable_vga, "Disable VGA resource access through vfio-pci");
	46	#endif
	47
	48	static bool disable_idle_d3;
	49	module_param(disable_idle_d3, bool, S_IRUGO \| S_IWUSR);
	50	MODULE_PARM_DESC(disable_idle_d3,
	51	"Disable using the PCI D3 low power state for idle, unused devices");
	52
ff53edf6 MG	53	static bool enable_sriov;
	54	#ifdef CONFIG_PCI_IOV
	55	module_param(enable_sriov, bool, 0644);
	56	MODULE_PARM_DESC(enable_sriov, "Enable support for SR-IOV configuration. Enabling SR-IOV on a PF typically requires support of the userspace PF driver, enabling VFs without such support may result in non-functional VFs or PF.");
	57	#endif
	58
	59	static bool disable_denylist;
	60	module_param(disable_denylist, bool, 0444);
	61	MODULE_PARM_DESC(disable_denylist, "Disable use of device denylist. Disabling the denylist allows binding to devices with known errata that may lead to exploitable stability or security issues when accessed by untrusted users.");
	62
	63	static bool vfio_pci_dev_in_denylist(struct pci_dev *pdev)
	64	{
	65	switch (pdev->vendor) {
	66	case PCI_VENDOR_ID_INTEL:
	67	switch (pdev->device) {
	68	case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
	69	case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF:
	70	case PCI_DEVICE_ID_INTEL_QAT_C62X:
	71	case PCI_DEVICE_ID_INTEL_QAT_C62X_VF:
	72	case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
	73	case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF:
95feb316 AV	74	case PCI_DEVICE_ID_INTEL_DSA_SPR0:
95feb316 AV	75	case PCI_DEVICE_ID_INTEL_IAX_SPR0:
ff53edf6 MG	76	return true;
	77	default:
	78	return false;
	79	}
	80	}
	81
	82	return false;
	83	}
	84
	85	static bool vfio_pci_is_denylisted(struct pci_dev *pdev)
	86	{
	87	if (!vfio_pci_dev_in_denylist(pdev))
	88	return false;
	89
	90	if (disable_denylist) {
	91	pci_warn(pdev,
	92	"device denylist disabled - allowing device %04x:%04x.\n",
	93	pdev->vendor, pdev->device);
	94	return false;
	95	}
	96
	97	pci_warn(pdev, "%04x:%04x exists in vfio-pci device denylist, driver probing disallowed.\n",
	98	pdev->vendor, pdev->device);
	99
	100	return true;
	101	}
	102
2fb89f56 MG	103	static int vfio_pci_open_device(struct vfio_device *core_vdev)
	104	{
	105	struct vfio_pci_core_device *vdev =
	106	container_of(core_vdev, struct vfio_pci_core_device, vdev);
	107	struct pci_dev *pdev = vdev->pdev;
	108	int ret;
	109
	110	ret = vfio_pci_core_enable(vdev);
	111	if (ret)
	112	return ret;
	113
41112160	114	if (vfio_pci_is_intel_display(pdev)) {
2fb89f56 MG	115	ret = vfio_pci_igd_init(vdev);
	116	if (ret && ret != -ENODEV) {
	117	pci_warn(pdev, "Failed to setup Intel IGD regions\n");
	118	vfio_pci_core_disable(vdev);
	119	return ret;
	120	}
	121	}
	122
	123	vfio_pci_core_finish_enable(vdev);
	124
	125	return 0;
	126	}
	127
ff53edf6 MG	128	static const struct vfio_device_ops vfio_pci_ops = {
ff53edf6 MG	129	.name = "vfio-pci",
63d7c779 YL	130	.init = vfio_pci_core_init_dev,
63d7c779 YL	131	.release = vfio_pci_core_release_dev,
2fb89f56	132	.open_device = vfio_pci_open_device,
ff53edf6 MG	133	.close_device = vfio_pci_core_close_device,
ff53edf6 MG	134	.ioctl = vfio_pci_core_ioctl,
445ad495	135	.device_feature = vfio_pci_core_ioctl_feature,
ff53edf6 MG	136	.read = vfio_pci_core_read,
	137	.write = vfio_pci_core_write,
	138	.mmap = vfio_pci_core_mmap,
	139	.request = vfio_pci_core_request,
	140	.match = vfio_pci_core_match,
a4d1f91d JG	141	.bind_iommufd = vfio_iommufd_physical_bind,
	142	.unbind_iommufd = vfio_iommufd_physical_unbind,
	143	.attach_ioas = vfio_iommufd_physical_attach_ioas,
9048c734	144	.detach_ioas = vfio_iommufd_physical_detach_ioas,
29064134 YL	145	.pasid_attach_ioas = vfio_iommufd_physical_pasid_attach_ioas,
29064134 YL	146	.pasid_detach_ioas = vfio_iommufd_physical_pasid_detach_ioas,
ff53edf6 MG	147	};
	148
	149	static int vfio_pci_probe(struct pci_dev pdev, const struct pci_device_id id)
	150	{
	151	struct vfio_pci_core_device *vdev;
	152	int ret;
	153
	154	if (vfio_pci_is_denylisted(pdev))
	155	return -EINVAL;
	156
63d7c779 YL	157	vdev = vfio_alloc_device(vfio_pci_core_device, vdev, &pdev->dev,
	158	&vfio_pci_ops);
	159	if (IS_ERR(vdev))
	160	return PTR_ERR(vdev);
ff53edf6	161
91be0bd6	162	dev_set_drvdata(&pdev->dev, vdev);
ff53edf6 MG	163	ret = vfio_pci_core_register_device(vdev);
ff53edf6 MG	164	if (ret)
63d7c779	165	goto out_put_vdev;
ff53edf6 MG	166	return 0;
ff53edf6 MG	167
63d7c779 YL	168	out_put_vdev:
63d7c779 YL	169	vfio_put_device(&vdev->vdev);
ff53edf6 MG	170	return ret;
	171	}
	172
	173	static void vfio_pci_remove(struct pci_dev *pdev)
	174	{
	175	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
	176
	177	vfio_pci_core_unregister_device(vdev);
63d7c779	178	vfio_put_device(&vdev->vdev);
ff53edf6 MG	179	}
	180
	181	static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
	182	{
ff806cbd JG	183	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
ff806cbd JG	184
ff53edf6 MG	185	if (!enable_sriov)
	186	return -ENOENT;
	187
ff806cbd	188	return vfio_pci_core_sriov_configure(vdev, nr_virtfn);
ff53edf6 MG	189	}
ff53edf6 MG	190
cc6711b0 MG	191	static const struct pci_device_id vfio_pci_table[] = {
	192	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_ANY_ID, PCI_ANY_ID) }, /* match all by default */
	193	{}
	194	};
	195
	196	MODULE_DEVICE_TABLE(pci, vfio_pci_table);
	197
ff53edf6 MG	198	static struct pci_driver vfio_pci_driver = {
ff53edf6 MG	199	.name = "vfio-pci",
cc6711b0	200	.id_table = vfio_pci_table,
ff53edf6 MG	201	.probe = vfio_pci_probe,
	202	.remove = vfio_pci_remove,
	203	.sriov_configure = vfio_pci_sriov_configure,
	204	.err_handler = &vfio_pci_core_err_handlers,
70693f47	205	.driver_managed_dma = true,
ff53edf6 MG	206	};
	207
	208	static void __init vfio_pci_fill_ids(void)
	209	{
	210	char p, id;
	211	int rc;
	212
	213	/* no ids passed actually */
	214	if (ids[0] == '\0')
	215	return;
	216
	217	/* add ids specified in the module parameter */
	218	p = ids;
	219	while ((id = strsep(&p, ","))) {
	220	unsigned int vendor, device, subvendor = PCI_ANY_ID,
	221	subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
	222	int fields;
	223
	224	if (!strlen(id))
	225	continue;
	226
	227	fields = sscanf(id, "%x:%x:%x:%x:%x:%x",
	228	&vendor, &device, &subvendor, &subdevice,
	229	&class, &class_mask);
	230
	231	if (fields < 2) {
	232	pr_warn("invalid id string \"%s\"\n", id);
	233	continue;
	234	}
	235
	236	rc = pci_add_dynid(&vfio_pci_driver, vendor, device,
	237	subvendor, subdevice, class, class_mask, 0);
	238	if (rc)
	239	pr_warn("failed to add dynamic id [%04x:%04x[%04x:%04x]] class %#08x/%08x (%d)\n",
	240	vendor, device, subvendor, subdevice,
	241	class, class_mask, rc);
	242	else
	243	pr_info("add [%04x:%04x[%04x:%04x]] class %#08x/%08x\n",
	244	vendor, device, subvendor, subdevice,
	245	class, class_mask);
	246	}
	247	}
	248
	249	static int __init vfio_pci_init(void)
	250	{
	251	int ret;
c61302aa YH	252	bool is_disable_vga = true;
	253
	254	#ifdef CONFIG_VFIO_PCI_VGA
	255	is_disable_vga = disable_vga;
	256	#endif
	257
	258	vfio_pci_core_set_params(nointxmask, is_disable_vga, disable_idle_d3);
ff53edf6	259
ff53edf6 MG	260	/* Register and scan for devices */
	261	ret = pci_register_driver(&vfio_pci_driver);
	262	if (ret)
7fa005ca	263	return ret;
ff53edf6 MG	264
	265	vfio_pci_fill_ids();
	266
	267	if (disable_denylist)
	268	pr_warn("device denylist disabled.\n");
	269
	270	return 0;
ff53edf6 MG	271	}
	272	module_init(vfio_pci_init);
	273
	274	static void __exit vfio_pci_cleanup(void)
	275	{
	276	pci_unregister_driver(&vfio_pci_driver);
ff53edf6 MG	277	}
	278	module_exit(vfio_pci_cleanup);
	279
	280	MODULE_LICENSE("GPL v2");
	281	MODULE_AUTHOR(DRIVER_AUTHOR);
	282	MODULE_DESCRIPTION(DRIVER_DESC);