[linux-block.git] / drivers / vfio / pci / vfio_pci_private.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
 *     Author: Alex Williamson <alex.williamson@redhat.com>
 *
 * Derived from original vfio:
 * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
 * Author: Tom Lyon, pugs@cisco.com
 */

#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/irqbypass.h>
#include <linux/types.h>
#include <linux/uuid.h>
#include <linux/notifier.h>

#ifndef VFIO_PCI_PRIVATE_H
#define VFIO_PCI_PRIVATE_H

#define VFIO_PCI_OFFSET_SHIFT   40

#define VFIO_PCI_OFFSET_TO_INDEX(off)	(off >> VFIO_PCI_OFFSET_SHIFT)
#define VFIO_PCI_INDEX_TO_OFFSET(index)	((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
#define VFIO_PCI_OFFSET_MASK	(((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)

/* Special capability IDs predefined access */
#define PCI_CAP_ID_INVALID		0xFF	/* default raw access */
#define PCI_CAP_ID_INVALID_VIRT		0xFE	/* default virt access */

/* Cap maximum number of ioeventfds per device (arbitrary) */
#define VFIO_PCI_IOEVENTFD_MAX		1000

struct vfio_pci_ioeventfd {
	struct list_head	next;
	struct vfio_pci_device	*vdev;
	struct virqfd		*virqfd;
	void __iomem		*addr;
	uint64_t		data;
	loff_t			pos;
	int			bar;
	int			count;
	bool			test_mem;
};

struct vfio_pci_irq_ctx {
	struct eventfd_ctx	*trigger;
	struct virqfd		*unmask;
	struct virqfd		*mask;
	char			*name;
	bool			masked;
	struct irq_bypass_producer	producer;
};

struct vfio_pci_device;
struct vfio_pci_region;

struct vfio_pci_regops {
	size_t	(*rw)(struct vfio_pci_device *vdev, char __user *buf,
		      size_t count, loff_t *ppos, bool iswrite);
	void	(*release)(struct vfio_pci_device *vdev,
			   struct vfio_pci_region *region);
	int	(*mmap)(struct vfio_pci_device *vdev,
			struct vfio_pci_region *region,
			struct vm_area_struct *vma);
	int	(*add_capability)(struct vfio_pci_device *vdev,
				  struct vfio_pci_region *region,
				  struct vfio_info_cap *caps);
};

struct vfio_pci_region {
	u32				type;
	u32				subtype;
	const struct vfio_pci_regops	*ops;
	void				*data;
	size_t				size;
	u32				flags;
};

struct vfio_pci_dummy_resource {
	struct resource		resource;
	int			index;
	struct list_head	res_next;
};

struct vfio_pci_reflck {
	struct kref		kref;
	struct mutex		lock;
};

struct vfio_pci_vf_token {
	struct mutex		lock;
	uuid_t			uuid;
	int			users;
};

struct vfio_pci_mmap_vma {
	struct vm_area_struct	*vma;
	struct list_head	vma_next;
};

struct vfio_pci_device {
	struct pci_dev		*pdev;
	void __iomem		*barmap[PCI_STD_NUM_BARS];
	bool			bar_mmap_supported[PCI_STD_NUM_BARS];
	u8			*pci_config_map;
	u8			*vconfig;
	struct perm_bits	*msi_perm;
	spinlock_t		irqlock;
	struct mutex		igate;
	struct vfio_pci_irq_ctx	*ctx;
	int			num_ctx;
	int			irq_type;
	int			num_regions;
	struct vfio_pci_region	*region;
	u8			msi_qmax;
	u8			msix_bar;
	u16			msix_size;
	u32			msix_offset;
	u32			rbar[7];
	bool			pci_2_3;
	bool			virq_disabled;
	bool			reset_works;
	bool			extended_caps;
	bool			bardirty;
	bool			has_vga;
	bool			needs_reset;
	bool			nointx;
	bool			needs_pm_restore;
	struct pci_saved_state	*pci_saved_state;
	struct pci_saved_state	*pm_save;
	struct vfio_pci_reflck	*reflck;
	int			refcnt;
	int			ioeventfds_nr;
	struct eventfd_ctx	*err_trigger;
	struct eventfd_ctx	*req_trigger;
	struct list_head	dummy_resources_list;
	struct mutex		ioeventfds_lock;
	struct list_head	ioeventfds_list;
	struct vfio_pci_vf_token	*vf_token;
	struct notifier_block	nb;
	struct mutex		vma_lock;
	struct list_head	vma_list;
	struct rw_semaphore	memory_lock;
};

#define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX)
#define is_msi(vdev) (vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX)
#define is_msix(vdev) (vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX)
#define is_irq_none(vdev) (!(is_intx(vdev) || is_msi(vdev) || is_msix(vdev)))
#define irq_is(vdev, type) (vdev->irq_type == type)

extern void vfio_pci_intx_mask(struct vfio_pci_device *vdev);
extern void vfio_pci_intx_unmask(struct vfio_pci_device *vdev);

extern int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev,
				   uint32_t flags, unsigned index,
				   unsigned start, unsigned count, void *data);

extern ssize_t vfio_pci_config_rw(struct vfio_pci_device *vdev,
				  char __user *buf, size_t count,
				  loff_t *ppos, bool iswrite);

extern ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf,
			       size_t count, loff_t *ppos, bool iswrite);

extern ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf,
			       size_t count, loff_t *ppos, bool iswrite);

extern long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
			       uint64_t data, int count, int fd);

extern int vfio_pci_init_perm_bits(void);
extern void vfio_pci_uninit_perm_bits(void);

extern int vfio_config_init(struct vfio_pci_device *vdev);
extern void vfio_config_free(struct vfio_pci_device *vdev);

extern int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
					unsigned int type, unsigned int subtype,
					const struct vfio_pci_regops *ops,
					size_t size, u32 flags, void *data);

extern int vfio_pci_set_power_state(struct vfio_pci_device *vdev,
				    pci_power_t state);

extern bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev);
extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device
						    *vdev);
extern u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev);
extern void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev,
					       u16 cmd);

#ifdef CONFIG_VFIO_PCI_IGD
extern int vfio_pci_igd_init(struct vfio_pci_device *vdev);
#else
static inline int vfio_pci_igd_init(struct vfio_pci_device *vdev)
{
	return -ENODEV;
}
#endif
#ifdef CONFIG_VFIO_PCI_NVLINK2
extern int vfio_pci_nvdia_v100_nvlink2_init(struct vfio_pci_device *vdev);
extern int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev);
#else
static inline int vfio_pci_nvdia_v100_nvlink2_init(struct vfio_pci_device *vdev)
{
	return -ENODEV;
}

static inline int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev)
{
	return -ENODEV;
}
#endif

#ifdef CONFIG_S390
extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
				       struct vfio_info_cap *caps);
#else
static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
					      struct vfio_info_cap *caps)
{
	return -ENODEV;
}
#endif

#endif /* VFIO_PCI_PRIVATE_H */
Commit	Line	Data
d2912cb1	1	/* SPDX-License-Identifier: GPL-2.0-only */
89e1f7d4 AW	2	/*
	3	* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
	4	* Author: Alex Williamson <alex.williamson@redhat.com>
	5	*
89e1f7d4 AW	6	* Derived from original vfio:
	7	* Copyright 2010 Cisco Systems, Inc. All rights reserved.
	8	* Author: Tom Lyon, pugs@cisco.com
	9	*/
	10
	11	#include <linux/mutex.h>
	12	#include <linux/pci.h>
6d7425f1	13	#include <linux/irqbypass.h>
28541d41	14	#include <linux/types.h>
cc20d799	15	#include <linux/uuid.h>
137e5531	16	#include <linux/notifier.h>
89e1f7d4 AW	17
	18	#ifndef VFIO_PCI_PRIVATE_H
	19	#define VFIO_PCI_PRIVATE_H
	20
	21	#define VFIO_PCI_OFFSET_SHIFT 40
	22
	23	#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT)
	24	#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
	25	#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
	26
345d7104 AW	27	/* Special capability IDs predefined access */
	28	#define PCI_CAP_ID_INVALID 0xFF /* default raw access */
	29	#define PCI_CAP_ID_INVALID_VIRT 0xFE /* default virt access */
	30
30656177 AW	31	/* Cap maximum number of ioeventfds per device (arbitrary) */
	32	#define VFIO_PCI_IOEVENTFD_MAX 1000
	33
	34	struct vfio_pci_ioeventfd {
	35	struct list_head next;
bc93b9ae	36	struct vfio_pci_device *vdev;
30656177 AW	37	struct virqfd *virqfd;
	38	void __iomem *addr;
	39	uint64_t data;
	40	loff_t pos;
	41	int bar;
	42	int count;
bc93b9ae	43	bool test_mem;
30656177 AW	44	};
30656177 AW	45
89e1f7d4 AW	46	struct vfio_pci_irq_ctx {
	47	struct eventfd_ctx *trigger;
	48	struct virqfd *unmask;
	49	struct virqfd *mask;
	50	char *name;
	51	bool masked;
6d7425f1	52	struct irq_bypass_producer producer;
89e1f7d4 AW	53	};
89e1f7d4 AW	54
28541d41 AW	55	struct vfio_pci_device;
	56	struct vfio_pci_region;
	57
	58	struct vfio_pci_regops {
	59	size_t (rw)(struct vfio_pci_device vdev, char __user *buf,
	60	size_t count, loff_t *ppos, bool iswrite);
	61	void (release)(struct vfio_pci_device vdev,
	62	struct vfio_pci_region *region);
a15b1883 AK	63	int (mmap)(struct vfio_pci_device vdev,
	64	struct vfio_pci_region *region,
	65	struct vm_area_struct *vma);
c2c0f1cd AK	66	int (add_capability)(struct vfio_pci_device vdev,
	67	struct vfio_pci_region *region,
	68	struct vfio_info_cap *caps);
28541d41 AW	69	};
	70
	71	struct vfio_pci_region {
	72	u32 type;
	73	u32 subtype;
	74	const struct vfio_pci_regops *ops;
	75	void *data;
	76	size_t size;
	77	u32 flags;
	78	};
	79
05f0c03f YX	80	struct vfio_pci_dummy_resource {
	81	struct resource resource;
	82	int index;
	83	struct list_head res_next;
	84	};
	85
e309df5b AW	86	struct vfio_pci_reflck {
	87	struct kref kref;
	88	struct mutex lock;
	89	};
	90
cc20d799 AW	91	struct vfio_pci_vf_token {
	92	struct mutex lock;
	93	uuid_t uuid;
	94	int users;
	95	};
	96
11c4cd07 AW	97	struct vfio_pci_mmap_vma {
	98	struct vm_area_struct *vma;
	99	struct list_head vma_next;
	100	};
	101
89e1f7d4 AW	102	struct vfio_pci_device {
89e1f7d4 AW	103	struct pci_dev *pdev;
c9c13ba4 DE	104	void __iomem *barmap[PCI_STD_NUM_BARS];
c9c13ba4 DE	105	bool bar_mmap_supported[PCI_STD_NUM_BARS];
89e1f7d4 AW	106	u8 *pci_config_map;
	107	u8 *vconfig;
	108	struct perm_bits *msi_perm;
	109	spinlock_t irqlock;
	110	struct mutex igate;
89e1f7d4 AW	111	struct vfio_pci_irq_ctx *ctx;
	112	int num_ctx;
	113	int irq_type;
28541d41 AW	114	int num_regions;
28541d41 AW	115	struct vfio_pci_region *region;
89e1f7d4 AW	116	u8 msi_qmax;
	117	u8 msix_bar;
	118	u16 msix_size;
	119	u32 msix_offset;
	120	u32 rbar[7];
	121	bool pci_2_3;
	122	bool virq_disabled;
	123	bool reset_works;
	124	bool extended_caps;
	125	bool bardirty;
84237a82	126	bool has_vga;
bc4fba77	127	bool needs_reset;
45074405	128	bool nointx;
51ef3a00	129	bool needs_pm_restore;
89e1f7d4	130	struct pci_saved_state *pci_saved_state;
51ef3a00	131	struct pci_saved_state *pm_save;
e309df5b	132	struct vfio_pci_reflck *reflck;
61d79256	133	int refcnt;
30656177	134	int ioeventfds_nr;
dad9f897	135	struct eventfd_ctx *err_trigger;
6140a8f5	136	struct eventfd_ctx *req_trigger;
05f0c03f	137	struct list_head dummy_resources_list;
30656177 AW	138	struct mutex ioeventfds_lock;
30656177 AW	139	struct list_head ioeventfds_list;
cc20d799	140	struct vfio_pci_vf_token *vf_token;
137e5531	141	struct notifier_block nb;
11c4cd07 AW	142	struct mutex vma_lock;
11c4cd07 AW	143	struct list_head vma_list;
abafbc55	144	struct rw_semaphore memory_lock;
89e1f7d4 AW	145	};
	146
	147	#define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX)
	148	#define is_msi(vdev) (vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX)
	149	#define is_msix(vdev) (vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX)
	150	#define is_irq_none(vdev) (!(is_intx(vdev) \|\| is_msi(vdev) \|\| is_msix(vdev)))
	151	#define irq_is(vdev, type) (vdev->irq_type == type)
	152
	153	extern void vfio_pci_intx_mask(struct vfio_pci_device *vdev);
	154	extern void vfio_pci_intx_unmask(struct vfio_pci_device *vdev);
	155
	156	extern int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev,
	157	uint32_t flags, unsigned index,
	158	unsigned start, unsigned count, void *data);
	159
906ee99d AW	160	extern ssize_t vfio_pci_config_rw(struct vfio_pci_device *vdev,
	161	char __user *buf, size_t count,
	162	loff_t *ppos, bool iswrite);
	163
	164	extern ssize_t vfio_pci_bar_rw(struct vfio_pci_device vdev, char __user buf,
	165	size_t count, loff_t *ppos, bool iswrite);
89e1f7d4	166
84237a82 AW	167	extern ssize_t vfio_pci_vga_rw(struct vfio_pci_device vdev, char __user buf,
	168	size_t count, loff_t *ppos, bool iswrite);
	169
30656177 AW	170	extern long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
	171	uint64_t data, int count, int fd);
	172
89e1f7d4 AW	173	extern int vfio_pci_init_perm_bits(void);
	174	extern void vfio_pci_uninit_perm_bits(void);
	175
89e1f7d4 AW	176	extern int vfio_config_init(struct vfio_pci_device *vdev);
89e1f7d4 AW	177	extern void vfio_config_free(struct vfio_pci_device *vdev);
28541d41 AW	178
	179	extern int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
	180	unsigned int type, unsigned int subtype,
	181	const struct vfio_pci_regops *ops,
	182	size_t size, u32 flags, void *data);
51ef3a00 AW	183
	184	extern int vfio_pci_set_power_state(struct vfio_pci_device *vdev,
	185	pci_power_t state);
	186
abafbc55 AW	187	extern bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev);
	188	extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device
	189	*vdev);
	190	extern u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev);
	191	extern void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev,
	192	u16 cmd);
	193
5846ff54	194	#ifdef CONFIG_VFIO_PCI_IGD
f572a960	195	extern int vfio_pci_igd_init(struct vfio_pci_device *vdev);
5846ff54	196	#else
f572a960	197	static inline int vfio_pci_igd_init(struct vfio_pci_device *vdev)
5846ff54 AW	198	{
	199	return -ENODEV;
	200	}
	201	#endif
7f928917 AK	202	#ifdef CONFIG_VFIO_PCI_NVLINK2
	203	extern int vfio_pci_nvdia_v100_nvlink2_init(struct vfio_pci_device *vdev);
	204	extern int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev);
	205	#else
	206	static inline int vfio_pci_nvdia_v100_nvlink2_init(struct vfio_pci_device *vdev)
	207	{
	208	return -ENODEV;
	209	}
	210
	211	static inline int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev)
	212	{
	213	return -ENODEV;
	214	}
	215	#endif
e6b817d4	216
b9abef43	217	#ifdef CONFIG_S390
e6b817d4 MR	218	extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
	219	struct vfio_info_cap *caps);
	220	#else
	221	static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
	222	struct vfio_info_cap *caps)
	223	{
	224	return -ENODEV;
	225	}
	226	#endif
	227
89e1f7d4	228	#endif /* VFIO_PCI_PRIVATE_H */