[linux-2.6-block.git] / drivers / pci / ats.c

// SPDX-License-Identifier: GPL-2.0
/*
 * PCI Express I/O Virtualization (IOV) support
 *   Address Translation Service 1.0
 *   Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com>
 *   PASID support added by Joerg Roedel <joerg.roedel@amd.com>
 *
 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
 * Copyright (C) 2011 Advanced Micro Devices,
 */

#include <linux/bitfield.h>
#include <linux/export.h>
#include <linux/pci-ats.h>
#include <linux/pci.h>
#include <linux/slab.h>

#include "pci.h"

void pci_ats_init(struct pci_dev *dev)
{
	int pos;

	if (pci_ats_disabled())
		return;

	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
	if (!pos)
		return;

	dev->ats_cap = pos;
}

/**
 * pci_ats_supported - check if the device can use ATS
 * @dev: the PCI device
 *
 * Returns true if the device supports ATS and is allowed to use it, false
 * otherwise.
 */
bool pci_ats_supported(struct pci_dev *dev)
{
	if (!dev->ats_cap)
		return false;

	return (dev->untrusted == 0);
}
EXPORT_SYMBOL_GPL(pci_ats_supported);

/**
 * pci_prepare_ats - Setup the PS for ATS
 * @dev: the PCI device
 * @ps: the IOMMU page shift
 *
 * This must be done by the IOMMU driver on the PF before any VFs are created to
 * ensure that the VF can have ATS enabled.
 *
 * Returns 0 on success, or negative on failure.
 */
int pci_prepare_ats(struct pci_dev *dev, int ps)
{
	u16 ctrl;

	if (!pci_ats_supported(dev))
		return -EINVAL;

	if (WARN_ON(dev->ats_enabled))
		return -EBUSY;

	if (ps < PCI_ATS_MIN_STU)
		return -EINVAL;

	if (dev->is_virtfn)
		return 0;

	dev->ats_stu = ps;
	ctrl = PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
	return 0;
}
EXPORT_SYMBOL_GPL(pci_prepare_ats);

/**
 * pci_enable_ats - enable the ATS capability
 * @dev: the PCI device
 * @ps: the IOMMU page shift
 *
 * Returns 0 on success, or negative on failure.
 */
int pci_enable_ats(struct pci_dev *dev, int ps)
{
	u16 ctrl;
	struct pci_dev *pdev;

	if (!pci_ats_supported(dev))
		return -EINVAL;

	if (WARN_ON(dev->ats_enabled))
		return -EBUSY;

	if (ps < PCI_ATS_MIN_STU)
		return -EINVAL;

	/*
	 * Note that enabling ATS on a VF fails unless it's already enabled
	 * with the same STU on the PF.
	 */
	ctrl = PCI_ATS_CTRL_ENABLE;
	if (dev->is_virtfn) {
		pdev = pci_physfn(dev);
		if (pdev->ats_stu != ps)
			return -EINVAL;
	} else {
		dev->ats_stu = ps;
		ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
	}
	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);

	dev->ats_enabled = 1;
	return 0;
}
EXPORT_SYMBOL_GPL(pci_enable_ats);

/**
 * pci_disable_ats - disable the ATS capability
 * @dev: the PCI device
 */
void pci_disable_ats(struct pci_dev *dev)
{
	u16 ctrl;

	if (WARN_ON(!dev->ats_enabled))
		return;

	pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);
	ctrl &= ~PCI_ATS_CTRL_ENABLE;
	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);

	dev->ats_enabled = 0;
}
EXPORT_SYMBOL_GPL(pci_disable_ats);

void pci_restore_ats_state(struct pci_dev *dev)
{
	u16 ctrl;

	if (!dev->ats_enabled)
		return;

	ctrl = PCI_ATS_CTRL_ENABLE;
	if (!dev->is_virtfn)
		ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
}

/**
 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth
 * @dev: the PCI device
 *
 * Returns the queue depth on success, or negative on failure.
 *
 * The ATS spec uses 0 in the Invalidate Queue Depth field to
 * indicate that the function can accept 32 Invalidate Request.
 * But here we use the `real' values (i.e. 1~32) for the Queue
 * Depth; and 0 indicates the function shares the Queue with
 * other functions (doesn't exclusively own a Queue).
 */
int pci_ats_queue_depth(struct pci_dev *dev)
{
	u16 cap;

	if (!dev->ats_cap)
		return -EINVAL;

	if (dev->is_virtfn)
		return 0;

	pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);
	return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP;
}

/**
 * pci_ats_page_aligned - Return Page Aligned Request bit status.
 * @pdev: the PCI device
 *
 * Returns 1, if the Untranslated Addresses generated by the device
 * are always aligned or 0 otherwise.
 *
 * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit
 * is set, it indicates the Untranslated Addresses generated by the
 * device are always aligned to a 4096 byte boundary.
 */
int pci_ats_page_aligned(struct pci_dev *pdev)
{
	u16 cap;

	if (!pdev->ats_cap)
		return 0;

	pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap);

	if (cap & PCI_ATS_CAP_PAGE_ALIGNED)
		return 1;

	return 0;
}

#ifdef CONFIG_PCI_PRI
void pci_pri_init(struct pci_dev *pdev)
{
	u16 status;

	pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);

	if (!pdev->pri_cap)
		return;

	pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status);
	if (status & PCI_PRI_STATUS_PASID)
		pdev->pasid_required = 1;
}

/**
 * pci_enable_pri - Enable PRI capability
 * @pdev: PCI device structure
 * @reqs: outstanding requests
 *
 * Returns 0 on success, negative value on error
 */
int pci_enable_pri(struct pci_dev *pdev, u32 reqs)
{
	u16 control, status;
	u32 max_requests;
	int pri = pdev->pri_cap;

	/*
	 * VFs must not implement the PRI Capability.  If their PF
	 * implements PRI, it is shared by the VFs, so if the PF PRI is
	 * enabled, it is also enabled for the VF.
	 */
	if (pdev->is_virtfn) {
		if (pci_physfn(pdev)->pri_enabled)
			return 0;
		return -EINVAL;
	}

	if (WARN_ON(pdev->pri_enabled))
		return -EBUSY;

	if (!pri)
		return -EINVAL;

	pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status);
	if (!(status & PCI_PRI_STATUS_STOPPED))
		return -EBUSY;

	pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests);
	reqs = min(max_requests, reqs);
	pdev->pri_reqs_alloc = reqs;
	pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);

	control = PCI_PRI_CTRL_ENABLE;
	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);

	pdev->pri_enabled = 1;

	return 0;
}

/**
 * pci_disable_pri - Disable PRI capability
 * @pdev: PCI device structure
 *
 * Only clears the enabled-bit, regardless of its former value
 */
void pci_disable_pri(struct pci_dev *pdev)
{
	u16 control;
	int pri = pdev->pri_cap;

	/* VFs share the PF PRI */
	if (pdev->is_virtfn)
		return;

	if (WARN_ON(!pdev->pri_enabled))
		return;

	if (!pri)
		return;

	pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control);
	control &= ~PCI_PRI_CTRL_ENABLE;
	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);

	pdev->pri_enabled = 0;
}
EXPORT_SYMBOL_GPL(pci_disable_pri);

/**
 * pci_restore_pri_state - Restore PRI
 * @pdev: PCI device structure
 */
void pci_restore_pri_state(struct pci_dev *pdev)
{
	u16 control = PCI_PRI_CTRL_ENABLE;
	u32 reqs = pdev->pri_reqs_alloc;
	int pri = pdev->pri_cap;

	if (pdev->is_virtfn)
		return;

	if (!pdev->pri_enabled)
		return;

	if (!pri)
		return;

	pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
}

/**
 * pci_reset_pri - Resets device's PRI state
 * @pdev: PCI device structure
 *
 * The PRI capability must be disabled before this function is called.
 * Returns 0 on success, negative value on error.
 */
int pci_reset_pri(struct pci_dev *pdev)
{
	u16 control;
	int pri = pdev->pri_cap;

	if (pdev->is_virtfn)
		return 0;

	if (WARN_ON(pdev->pri_enabled))
		return -EBUSY;

	if (!pri)
		return -EINVAL;

	control = PCI_PRI_CTRL_RESET;
	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);

	return 0;
}

/**
 * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit
 *				 status.
 * @pdev: PCI device structure
 *
 * Returns 1 if PASID is required in PRG Response Message, 0 otherwise.
 */
int pci_prg_resp_pasid_required(struct pci_dev *pdev)
{
	if (pdev->is_virtfn)
		pdev = pci_physfn(pdev);

	return pdev->pasid_required;
}

/**
 * pci_pri_supported - Check if PRI is supported.
 * @pdev: PCI device structure
 *
 * Returns true if PRI capability is present, false otherwise.
 */
bool pci_pri_supported(struct pci_dev *pdev)
{
	/* VFs share the PF PRI */
	if (pci_physfn(pdev)->pri_cap)
		return true;
	return false;
}
EXPORT_SYMBOL_GPL(pci_pri_supported);
#endif /* CONFIG_PCI_PRI */

#ifdef CONFIG_PCI_PASID
void pci_pasid_init(struct pci_dev *pdev)
{
	pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
}

/**
 * pci_enable_pasid - Enable the PASID capability
 * @pdev: PCI device structure
 * @features: Features to enable
 *
 * Returns 0 on success, negative value on error. This function checks
 * whether the features are actually supported by the device and returns
 * an error if not.
 */
int pci_enable_pasid(struct pci_dev *pdev, int features)
{
	u16 control, supported;
	int pasid = pdev->pasid_cap;

	/*
	 * VFs must not implement the PASID Capability, but if a PF
	 * supports PASID, its VFs share the PF PASID configuration.
	 */
	if (pdev->is_virtfn) {
		if (pci_physfn(pdev)->pasid_enabled)
			return 0;
		return -EINVAL;
	}

	if (WARN_ON(pdev->pasid_enabled))
		return -EBUSY;

	if (!pdev->eetlp_prefix_max && !pdev->pasid_no_tlp)
		return -EINVAL;

	if (!pasid)
		return -EINVAL;

	if (!pci_acs_path_enabled(pdev, NULL, PCI_ACS_RR | PCI_ACS_UF))
		return -EINVAL;

	pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
	supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;

	/* User wants to enable anything unsupported? */
	if ((supported & features) != features)
		return -EINVAL;

	control = PCI_PASID_CTRL_ENABLE | features;
	pdev->pasid_features = features;

	pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);

	pdev->pasid_enabled = 1;

	return 0;
}
EXPORT_SYMBOL_GPL(pci_enable_pasid);

/**
 * pci_disable_pasid - Disable the PASID capability
 * @pdev: PCI device structure
 */
void pci_disable_pasid(struct pci_dev *pdev)
{
	u16 control = 0;
	int pasid = pdev->pasid_cap;

	/* VFs share the PF PASID configuration */
	if (pdev->is_virtfn)
		return;

	if (WARN_ON(!pdev->pasid_enabled))
		return;

	if (!pasid)
		return;

	pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);

	pdev->pasid_enabled = 0;
}
EXPORT_SYMBOL_GPL(pci_disable_pasid);

/**
 * pci_restore_pasid_state - Restore PASID capabilities
 * @pdev: PCI device structure
 */
void pci_restore_pasid_state(struct pci_dev *pdev)
{
	u16 control;
	int pasid = pdev->pasid_cap;

	if (pdev->is_virtfn)
		return;

	if (!pdev->pasid_enabled)
		return;

	if (!pasid)
		return;

	control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features;
	pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
}

/**
 * pci_pasid_features - Check which PASID features are supported
 * @pdev: PCI device structure
 *
 * Return a negative value when no PASID capability is present.
 * Otherwise return a bitmask with supported features. Current
 * features reported are:
 * PCI_PASID_CAP_EXEC - Execute permission supported
 * PCI_PASID_CAP_PRIV - Privileged mode supported
 */
int pci_pasid_features(struct pci_dev *pdev)
{
	u16 supported;
	int pasid;

	if (pdev->is_virtfn)
		pdev = pci_physfn(pdev);

	pasid = pdev->pasid_cap;
	if (!pasid)
		return -EINVAL;

	pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);

	supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;

	return supported;
}
EXPORT_SYMBOL_GPL(pci_pasid_features);

/**
 * pci_max_pasids - Get maximum number of PASIDs supported by device
 * @pdev: PCI device structure
 *
 * Returns negative value when PASID capability is not present.
 * Otherwise it returns the number of supported PASIDs.
 */
int pci_max_pasids(struct pci_dev *pdev)
{
	u16 supported;
	int pasid;

	if (pdev->is_virtfn)
		pdev = pci_physfn(pdev);

	pasid = pdev->pasid_cap;
	if (!pasid)
		return -EINVAL;

	pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);

	return (1 << FIELD_GET(PCI_PASID_CAP_WIDTH, supported));
}
EXPORT_SYMBOL_GPL(pci_max_pasids);

/**
 * pci_pasid_status - Check the PASID status
 * @pdev: PCI device structure
 *
 * Returns a negative value when no PASID capability is present.
 * Otherwise the value of the control register is returned.
 * Status reported are:
 *
 * PCI_PASID_CTRL_ENABLE - PASID enabled
 * PCI_PASID_CTRL_EXEC - Execute permission enabled
 * PCI_PASID_CTRL_PRIV - Privileged mode enabled
 */
int pci_pasid_status(struct pci_dev *pdev)
{
	int pasid;
	u16 ctrl;

	if (pdev->is_virtfn)
		pdev = pci_physfn(pdev);

	pasid = pdev->pasid_cap;
	if (!pasid)
		return -EINVAL;

	pci_read_config_word(pdev, pasid + PCI_PASID_CTRL, &ctrl);

	ctrl &= PCI_PASID_CTRL_ENABLE | PCI_PASID_CTRL_EXEC |
		PCI_PASID_CTRL_PRIV;

	return ctrl;
}
EXPORT_SYMBOL_GPL(pci_pasid_status);
#endif /* CONFIG_PCI_PASID */
Commit	Line	Data
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* PCI Express I/O Virtualization (IOV) support
	4	* Address Translation Service 1.0
	5	* Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com>
	6	* PASID support added by Joerg Roedel <joerg.roedel@amd.com>
	7	*
	8	* Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
	9	* Copyright (C) 2011 Advanced Micro Devices,
	10	*/
	11
	12	#include <linux/bitfield.h>
	13	#include <linux/export.h>
	14	#include <linux/pci-ats.h>
	15	#include <linux/pci.h>
	16	#include <linux/slab.h>
	17
	18	#include "pci.h"
	19
	20	void pci_ats_init(struct pci_dev *dev)
	21	{
	22	int pos;
	23
	24	if (pci_ats_disabled())
	25	return;
	26
	27	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
	28	if (!pos)
	29	return;
	30
	31	dev->ats_cap = pos;
	32	}
	33
	34	/**
	35	* pci_ats_supported - check if the device can use ATS
	36	* @dev: the PCI device
	37	*
	38	* Returns true if the device supports ATS and is allowed to use it, false
	39	* otherwise.
	40	*/
	41	bool pci_ats_supported(struct pci_dev *dev)
	42	{
	43	if (!dev->ats_cap)
	44	return false;
	45
	46	return (dev->untrusted == 0);
	47	}
	48	EXPORT_SYMBOL_GPL(pci_ats_supported);
	49
	50	/**
	51	* pci_prepare_ats - Setup the PS for ATS
	52	* @dev: the PCI device
	53	* @ps: the IOMMU page shift
	54	*
	55	* This must be done by the IOMMU driver on the PF before any VFs are created to
	56	* ensure that the VF can have ATS enabled.
	57	*
	58	* Returns 0 on success, or negative on failure.
	59	*/
	60	int pci_prepare_ats(struct pci_dev *dev, int ps)
	61	{
	62	u16 ctrl;
	63
	64	if (!pci_ats_supported(dev))
	65	return -EINVAL;
	66
	67	if (WARN_ON(dev->ats_enabled))
	68	return -EBUSY;
	69
	70	if (ps < PCI_ATS_MIN_STU)
	71	return -EINVAL;
	72
	73	if (dev->is_virtfn)
	74	return 0;
	75
	76	dev->ats_stu = ps;
	77	ctrl = PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
	78	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
	79	return 0;
	80	}
	81	EXPORT_SYMBOL_GPL(pci_prepare_ats);
	82
	83	/**
	84	* pci_enable_ats - enable the ATS capability
	85	* @dev: the PCI device
	86	* @ps: the IOMMU page shift
	87	*
	88	* Returns 0 on success, or negative on failure.
	89	*/
	90	int pci_enable_ats(struct pci_dev *dev, int ps)
	91	{
	92	u16 ctrl;
	93	struct pci_dev *pdev;
	94
	95	if (!pci_ats_supported(dev))
	96	return -EINVAL;
	97
	98	if (WARN_ON(dev->ats_enabled))
	99	return -EBUSY;
	100
	101	if (ps < PCI_ATS_MIN_STU)
	102	return -EINVAL;
	103
	104	/*
	105	* Note that enabling ATS on a VF fails unless it's already enabled
	106	* with the same STU on the PF.
	107	*/
	108	ctrl = PCI_ATS_CTRL_ENABLE;
	109	if (dev->is_virtfn) {
	110	pdev = pci_physfn(dev);
	111	if (pdev->ats_stu != ps)
	112	return -EINVAL;
	113	} else {
	114	dev->ats_stu = ps;
	115	ctrl \|= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
	116	}
	117	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
	118
	119	dev->ats_enabled = 1;
	120	return 0;
	121	}
	122	EXPORT_SYMBOL_GPL(pci_enable_ats);
	123
	124	/**
	125	* pci_disable_ats - disable the ATS capability
	126	* @dev: the PCI device
	127	*/
	128	void pci_disable_ats(struct pci_dev *dev)
	129	{
	130	u16 ctrl;
	131
	132	if (WARN_ON(!dev->ats_enabled))
	133	return;
	134
	135	pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);
	136	ctrl &= ~PCI_ATS_CTRL_ENABLE;
	137	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
	138
	139	dev->ats_enabled = 0;
	140	}
	141	EXPORT_SYMBOL_GPL(pci_disable_ats);
	142
	143	void pci_restore_ats_state(struct pci_dev *dev)
	144	{
	145	u16 ctrl;
	146
	147	if (!dev->ats_enabled)
	148	return;
	149
	150	ctrl = PCI_ATS_CTRL_ENABLE;
	151	if (!dev->is_virtfn)
	152	ctrl \|= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
	153	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
	154	}
	155
	156	/**
	157	* pci_ats_queue_depth - query the ATS Invalidate Queue Depth
	158	* @dev: the PCI device
	159	*
	160	* Returns the queue depth on success, or negative on failure.
	161	*
	162	* The ATS spec uses 0 in the Invalidate Queue Depth field to
	163	* indicate that the function can accept 32 Invalidate Request.
	164	* But here we use the `real' values (i.e. 1~32) for the Queue
	165	* Depth; and 0 indicates the function shares the Queue with
	166	* other functions (doesn't exclusively own a Queue).
	167	*/
	168	int pci_ats_queue_depth(struct pci_dev *dev)
	169	{
	170	u16 cap;
	171
	172	if (!dev->ats_cap)
	173	return -EINVAL;
	174
	175	if (dev->is_virtfn)
	176	return 0;
	177
	178	pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);
	179	return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP;
	180	}
	181
	182	/**
	183	* pci_ats_page_aligned - Return Page Aligned Request bit status.
	184	* @pdev: the PCI device
	185	*
	186	* Returns 1, if the Untranslated Addresses generated by the device
	187	* are always aligned or 0 otherwise.
	188	*
	189	* Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit
	190	* is set, it indicates the Untranslated Addresses generated by the
	191	* device are always aligned to a 4096 byte boundary.
	192	*/
	193	int pci_ats_page_aligned(struct pci_dev *pdev)
	194	{
	195	u16 cap;
	196
	197	if (!pdev->ats_cap)
	198	return 0;
	199
	200	pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap);
	201
	202	if (cap & PCI_ATS_CAP_PAGE_ALIGNED)
	203	return 1;
	204
	205	return 0;
	206	}
	207
	208	#ifdef CONFIG_PCI_PRI
	209	void pci_pri_init(struct pci_dev *pdev)
	210	{
	211	u16 status;
	212
	213	pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
	214
	215	if (!pdev->pri_cap)
	216	return;
	217
	218	pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status);
	219	if (status & PCI_PRI_STATUS_PASID)
	220	pdev->pasid_required = 1;
	221	}
	222
	223	/**
	224	* pci_enable_pri - Enable PRI capability
	225	* @pdev: PCI device structure
	226	* @reqs: outstanding requests
	227	*
	228	* Returns 0 on success, negative value on error
	229	*/
	230	int pci_enable_pri(struct pci_dev *pdev, u32 reqs)
	231	{
	232	u16 control, status;
	233	u32 max_requests;
	234	int pri = pdev->pri_cap;
	235
	236	/*
	237	* VFs must not implement the PRI Capability. If their PF
	238	* implements PRI, it is shared by the VFs, so if the PF PRI is
	239	* enabled, it is also enabled for the VF.
	240	*/
	241	if (pdev->is_virtfn) {
	242	if (pci_physfn(pdev)->pri_enabled)
	243	return 0;
	244	return -EINVAL;
	245	}
	246
	247	if (WARN_ON(pdev->pri_enabled))
	248	return -EBUSY;
	249
	250	if (!pri)
	251	return -EINVAL;
	252
	253	pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status);
	254	if (!(status & PCI_PRI_STATUS_STOPPED))
	255	return -EBUSY;
	256
	257	pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests);
	258	reqs = min(max_requests, reqs);
	259	pdev->pri_reqs_alloc = reqs;
	260	pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
	261
	262	control = PCI_PRI_CTRL_ENABLE;
	263	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
	264
	265	pdev->pri_enabled = 1;
	266
	267	return 0;
	268	}
	269
	270	/**
	271	* pci_disable_pri - Disable PRI capability
	272	* @pdev: PCI device structure
	273	*
	274	* Only clears the enabled-bit, regardless of its former value
	275	*/
	276	void pci_disable_pri(struct pci_dev *pdev)
	277	{
	278	u16 control;
	279	int pri = pdev->pri_cap;
	280
	281	/* VFs share the PF PRI */
	282	if (pdev->is_virtfn)
	283	return;
	284
	285	if (WARN_ON(!pdev->pri_enabled))
	286	return;
	287
	288	if (!pri)
	289	return;
	290
	291	pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control);
	292	control &= ~PCI_PRI_CTRL_ENABLE;
	293	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
	294
	295	pdev->pri_enabled = 0;
	296	}
	297	EXPORT_SYMBOL_GPL(pci_disable_pri);
	298
	299	/**
	300	* pci_restore_pri_state - Restore PRI
	301	* @pdev: PCI device structure
	302	*/
	303	void pci_restore_pri_state(struct pci_dev *pdev)
	304	{
	305	u16 control = PCI_PRI_CTRL_ENABLE;
	306	u32 reqs = pdev->pri_reqs_alloc;
	307	int pri = pdev->pri_cap;
	308
	309	if (pdev->is_virtfn)
	310	return;
	311
	312	if (!pdev->pri_enabled)
	313	return;
	314
	315	if (!pri)
	316	return;
	317
	318	pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
	319	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
	320	}
	321
	322	/**
	323	* pci_reset_pri - Resets device's PRI state
	324	* @pdev: PCI device structure
	325	*
	326	* The PRI capability must be disabled before this function is called.
	327	* Returns 0 on success, negative value on error.
	328	*/
	329	int pci_reset_pri(struct pci_dev *pdev)
	330	{
	331	u16 control;
	332	int pri = pdev->pri_cap;
	333
	334	if (pdev->is_virtfn)
	335	return 0;
	336
	337	if (WARN_ON(pdev->pri_enabled))
	338	return -EBUSY;
	339
	340	if (!pri)
	341	return -EINVAL;
	342
	343	control = PCI_PRI_CTRL_RESET;
	344	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
	345
	346	return 0;
	347	}
	348
	349	/**
	350	* pci_prg_resp_pasid_required - Return PRG Response PASID Required bit
	351	* status.
	352	* @pdev: PCI device structure
	353	*
	354	* Returns 1 if PASID is required in PRG Response Message, 0 otherwise.
	355	*/
	356	int pci_prg_resp_pasid_required(struct pci_dev *pdev)
	357	{
	358	if (pdev->is_virtfn)
	359	pdev = pci_physfn(pdev);
	360
	361	return pdev->pasid_required;
	362	}
	363
	364	/**
	365	* pci_pri_supported - Check if PRI is supported.
	366	* @pdev: PCI device structure
	367	*
	368	* Returns true if PRI capability is present, false otherwise.
	369	*/
	370	bool pci_pri_supported(struct pci_dev *pdev)
	371	{
	372	/* VFs share the PF PRI */
	373	if (pci_physfn(pdev)->pri_cap)
	374	return true;
	375	return false;
	376	}
	377	EXPORT_SYMBOL_GPL(pci_pri_supported);
	378	#endif /* CONFIG_PCI_PRI */
	379
	380	#ifdef CONFIG_PCI_PASID
	381	void pci_pasid_init(struct pci_dev *pdev)
	382	{
	383	pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
	384	}
	385
	386	/**
	387	* pci_enable_pasid - Enable the PASID capability
	388	* @pdev: PCI device structure
	389	* @features: Features to enable
	390	*
	391	* Returns 0 on success, negative value on error. This function checks
	392	* whether the features are actually supported by the device and returns
	393	* an error if not.
	394	*/
	395	int pci_enable_pasid(struct pci_dev *pdev, int features)
	396	{
	397	u16 control, supported;
	398	int pasid = pdev->pasid_cap;
	399
	400	/*
	401	* VFs must not implement the PASID Capability, but if a PF
	402	* supports PASID, its VFs share the PF PASID configuration.
	403	*/
	404	if (pdev->is_virtfn) {
	405	if (pci_physfn(pdev)->pasid_enabled)
	406	return 0;
	407	return -EINVAL;
	408	}
	409
	410	if (WARN_ON(pdev->pasid_enabled))
	411	return -EBUSY;
	412
	413	if (!pdev->eetlp_prefix_max && !pdev->pasid_no_tlp)
	414	return -EINVAL;
	415
	416	if (!pasid)
	417	return -EINVAL;
	418
	419	if (!pci_acs_path_enabled(pdev, NULL, PCI_ACS_RR \| PCI_ACS_UF))
	420	return -EINVAL;
	421
	422	pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
	423	supported &= PCI_PASID_CAP_EXEC \| PCI_PASID_CAP_PRIV;
	424
	425	/* User wants to enable anything unsupported? */
	426	if ((supported & features) != features)
	427	return -EINVAL;
	428
	429	control = PCI_PASID_CTRL_ENABLE \| features;
	430	pdev->pasid_features = features;
	431
	432	pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
	433
	434	pdev->pasid_enabled = 1;
	435
	436	return 0;
	437	}
	438	EXPORT_SYMBOL_GPL(pci_enable_pasid);
	439
	440	/**
	441	* pci_disable_pasid - Disable the PASID capability
	442	* @pdev: PCI device structure
	443	*/
	444	void pci_disable_pasid(struct pci_dev *pdev)
	445	{
	446	u16 control = 0;
	447	int pasid = pdev->pasid_cap;
	448
	449	/* VFs share the PF PASID configuration */
	450	if (pdev->is_virtfn)
	451	return;
	452
	453	if (WARN_ON(!pdev->pasid_enabled))
	454	return;
	455
	456	if (!pasid)
	457	return;
	458
	459	pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
	460
	461	pdev->pasid_enabled = 0;
	462	}
	463	EXPORT_SYMBOL_GPL(pci_disable_pasid);
	464
	465	/**
	466	* pci_restore_pasid_state - Restore PASID capabilities
	467	* @pdev: PCI device structure
	468	*/
	469	void pci_restore_pasid_state(struct pci_dev *pdev)
	470	{
	471	u16 control;
	472	int pasid = pdev->pasid_cap;
	473
	474	if (pdev->is_virtfn)
	475	return;
	476
	477	if (!pdev->pasid_enabled)
	478	return;
	479
	480	if (!pasid)
	481	return;
	482
	483	control = PCI_PASID_CTRL_ENABLE \| pdev->pasid_features;
	484	pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
	485	}
	486
	487	/**
	488	* pci_pasid_features - Check which PASID features are supported
	489	* @pdev: PCI device structure
	490	*
	491	* Return a negative value when no PASID capability is present.
	492	* Otherwise return a bitmask with supported features. Current
	493	* features reported are:
	494	* PCI_PASID_CAP_EXEC - Execute permission supported
	495	* PCI_PASID_CAP_PRIV - Privileged mode supported
	496	*/
	497	int pci_pasid_features(struct pci_dev *pdev)
	498	{
	499	u16 supported;
	500	int pasid;
	501
	502	if (pdev->is_virtfn)
	503	pdev = pci_physfn(pdev);
	504
	505	pasid = pdev->pasid_cap;
	506	if (!pasid)
	507	return -EINVAL;
	508
	509	pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
	510
	511	supported &= PCI_PASID_CAP_EXEC \| PCI_PASID_CAP_PRIV;
	512
	513	return supported;
	514	}
	515	EXPORT_SYMBOL_GPL(pci_pasid_features);
	516
	517	/**
	518	* pci_max_pasids - Get maximum number of PASIDs supported by device
	519	* @pdev: PCI device structure
	520	*
	521	* Returns negative value when PASID capability is not present.
	522	* Otherwise it returns the number of supported PASIDs.
	523	*/
	524	int pci_max_pasids(struct pci_dev *pdev)
	525	{
	526	u16 supported;
	527	int pasid;
	528
	529	if (pdev->is_virtfn)
	530	pdev = pci_physfn(pdev);
	531
	532	pasid = pdev->pasid_cap;
	533	if (!pasid)
	534	return -EINVAL;
	535
	536	pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
	537
	538	return (1 << FIELD_GET(PCI_PASID_CAP_WIDTH, supported));
	539	}
	540	EXPORT_SYMBOL_GPL(pci_max_pasids);
	541
	542	/**
	543	* pci_pasid_status - Check the PASID status
	544	* @pdev: PCI device structure
	545	*
	546	* Returns a negative value when no PASID capability is present.
	547	* Otherwise the value of the control register is returned.
	548	* Status reported are:
	549	*
	550	* PCI_PASID_CTRL_ENABLE - PASID enabled
	551	* PCI_PASID_CTRL_EXEC - Execute permission enabled
	552	* PCI_PASID_CTRL_PRIV - Privileged mode enabled
	553	*/
	554	int pci_pasid_status(struct pci_dev *pdev)
	555	{
	556	int pasid;
	557	u16 ctrl;
	558
	559	if (pdev->is_virtfn)
	560	pdev = pci_physfn(pdev);
	561
	562	pasid = pdev->pasid_cap;
	563	if (!pasid)
	564	return -EINVAL;
	565
	566	pci_read_config_word(pdev, pasid + PCI_PASID_CTRL, &ctrl);
	567
	568	ctrl &= PCI_PASID_CTRL_ENABLE \| PCI_PASID_CTRL_EXEC \|
	569	PCI_PASID_CTRL_PRIV;
	570
	571	return ctrl;
	572	}
	573	EXPORT_SYMBOL_GPL(pci_pasid_status);
	574	#endif /* CONFIG_PCI_PASID */