[linux-2.6-block.git] / drivers / vfio / virqfd.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * VFIO generic eventfd code for IRQFD support.
 * Derived from drivers/vfio/pci/vfio_pci_intrs.c
 *
 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
 *     Author: Alex Williamson <alex.williamson@redhat.com>
 */

#include <linux/vfio.h>
#include <linux/eventfd.h>
#include <linux/file.h>
#include <linux/module.h>
#include <linux/slab.h>

#define DRIVER_VERSION  "0.1"
#define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
#define DRIVER_DESC     "IRQFD support for VFIO bus drivers"

static struct workqueue_struct *vfio_irqfd_cleanup_wq;
static DEFINE_SPINLOCK(virqfd_lock);

static int __init vfio_virqfd_init(void)
{
	vfio_irqfd_cleanup_wq =
		create_singlethread_workqueue("vfio-irqfd-cleanup");
	if (!vfio_irqfd_cleanup_wq)
		return -ENOMEM;

	return 0;
}

static void __exit vfio_virqfd_exit(void)
{
	destroy_workqueue(vfio_irqfd_cleanup_wq);
}

static void virqfd_deactivate(struct virqfd *virqfd)
{
	queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown);
}

static int virqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
{
	struct virqfd *virqfd = container_of(wait, struct virqfd, wait);
	__poll_t flags = key_to_poll(key);

	if (flags & EPOLLIN) {
		u64 cnt;
		eventfd_ctx_do_read(virqfd->eventfd, &cnt);

		/* An event has been signaled, call function */
		if ((!virqfd->handler ||
		     virqfd->handler(virqfd->opaque, virqfd->data)) &&
		    virqfd->thread)
			schedule_work(&virqfd->inject);
	}

	if (flags & EPOLLHUP) {
		unsigned long flags;
		spin_lock_irqsave(&virqfd_lock, flags);

		/*
		 * The eventfd is closing, if the virqfd has not yet been
		 * queued for release, as determined by testing whether the
		 * virqfd pointer to it is still valid, queue it now.  As
		 * with kvm irqfds, we know we won't race against the virqfd
		 * going away because we hold the lock to get here.
		 */
		if (*(virqfd->pvirqfd) == virqfd) {
			*(virqfd->pvirqfd) = NULL;
			virqfd_deactivate(virqfd);
		}

		spin_unlock_irqrestore(&virqfd_lock, flags);
	}

	return 0;
}

static void virqfd_ptable_queue_proc(struct file *file,
				     wait_queue_head_t *wqh, poll_table *pt)
{
	struct virqfd *virqfd = container_of(pt, struct virqfd, pt);
	add_wait_queue(wqh, &virqfd->wait);
}

static void virqfd_shutdown(struct work_struct *work)
{
	struct virqfd *virqfd = container_of(work, struct virqfd, shutdown);
	u64 cnt;

	eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt);
	flush_work(&virqfd->inject);
	eventfd_ctx_put(virqfd->eventfd);

	kfree(virqfd);
}

static void virqfd_inject(struct work_struct *work)
{
	struct virqfd *virqfd = container_of(work, struct virqfd, inject);
	if (virqfd->thread)
		virqfd->thread(virqfd->opaque, virqfd->data);
}

int vfio_virqfd_enable(void *opaque,
		       int (*handler)(void *, void *),
		       void (*thread)(void *, void *),
		       void *data, struct virqfd **pvirqfd, int fd)
{
	struct fd irqfd;
	struct eventfd_ctx *ctx;
	struct virqfd *virqfd;
	int ret = 0;
	__poll_t events;

	virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL);
	if (!virqfd)
		return -ENOMEM;

	virqfd->pvirqfd = pvirqfd;
	virqfd->opaque = opaque;
	virqfd->handler = handler;
	virqfd->thread = thread;
	virqfd->data = data;

	INIT_WORK(&virqfd->shutdown, virqfd_shutdown);
	INIT_WORK(&virqfd->inject, virqfd_inject);

	irqfd = fdget(fd);
	if (!irqfd.file) {
		ret = -EBADF;
		goto err_fd;
	}

	ctx = eventfd_ctx_fileget(irqfd.file);
	if (IS_ERR(ctx)) {
		ret = PTR_ERR(ctx);
		goto err_ctx;
	}

	virqfd->eventfd = ctx;

	/*
	 * virqfds can be released by closing the eventfd or directly
	 * through ioctl.  These are both done through a workqueue, so
	 * we update the pointer to the virqfd under lock to avoid
	 * pushing multiple jobs to release the same virqfd.
	 */
	spin_lock_irq(&virqfd_lock);

	if (*pvirqfd) {
		spin_unlock_irq(&virqfd_lock);
		ret = -EBUSY;
		goto err_busy;
	}
	*pvirqfd = virqfd;

	spin_unlock_irq(&virqfd_lock);

	/*
	 * Install our own custom wake-up handling so we are notified via
	 * a callback whenever someone signals the underlying eventfd.
	 */
	init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup);
	init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc);

	events = vfs_poll(irqfd.file, &virqfd->pt);

	/*
	 * Check if there was an event already pending on the eventfd
	 * before we registered and trigger it as if we didn't miss it.
	 */
	if (events & EPOLLIN) {
		if ((!handler || handler(opaque, data)) && thread)
			schedule_work(&virqfd->inject);
	}

	/*
	 * Do not drop the file until the irqfd is fully initialized,
	 * otherwise we might race against the EPOLLHUP.
	 */
	fdput(irqfd);

	return 0;
err_busy:
	eventfd_ctx_put(ctx);
err_ctx:
	fdput(irqfd);
err_fd:
	kfree(virqfd);

	return ret;
}
EXPORT_SYMBOL_GPL(vfio_virqfd_enable);

void vfio_virqfd_disable(struct virqfd **pvirqfd)
{
	unsigned long flags;

	spin_lock_irqsave(&virqfd_lock, flags);

	if (*pvirqfd) {
		virqfd_deactivate(*pvirqfd);
		*pvirqfd = NULL;
	}

	spin_unlock_irqrestore(&virqfd_lock, flags);

	/*
	 * Block until we know all outstanding shutdown jobs have completed.
	 * Even if we don't queue the job, flush the wq to be sure it's
	 * been released.
	 */
	flush_workqueue(vfio_irqfd_cleanup_wq);
}
EXPORT_SYMBOL_GPL(vfio_virqfd_disable);

module_init(vfio_virqfd_init);
module_exit(vfio_virqfd_exit);

MODULE_VERSION(DRIVER_VERSION);
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);
Commit	Line	Data
d2912cb1	1	// SPDX-License-Identifier: GPL-2.0-only
7e992d69 AM	2	/*
	3	* VFIO generic eventfd code for IRQFD support.
	4	* Derived from drivers/vfio/pci/vfio_pci_intrs.c
	5	*
	6	* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
	7	* Author: Alex Williamson <alex.williamson@redhat.com>
7e992d69 AM	8	*/
	9
	10	#include <linux/vfio.h>
	11	#include <linux/eventfd.h>
	12	#include <linux/file.h>
71be3423	13	#include <linux/module.h>
7e992d69 AM	14	#include <linux/slab.h>
7e992d69 AM	15
71be3423 AW	16	#define DRIVER_VERSION "0.1"
	17	#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
	18	#define DRIVER_DESC "IRQFD support for VFIO bus drivers"
	19
7e992d69	20	static struct workqueue_struct *vfio_irqfd_cleanup_wq;
66fdc052	21	static DEFINE_SPINLOCK(virqfd_lock);
7e992d69	22
71be3423	23	static int __init vfio_virqfd_init(void)
7e992d69 AM	24	{
	25	vfio_irqfd_cleanup_wq =
	26	create_singlethread_workqueue("vfio-irqfd-cleanup");
	27	if (!vfio_irqfd_cleanup_wq)
	28	return -ENOMEM;
	29
	30	return 0;
	31	}
	32
71be3423	33	static void __exit vfio_virqfd_exit(void)
7e992d69 AM	34	{
	35	destroy_workqueue(vfio_irqfd_cleanup_wq);
	36	}
	37
	38	static void virqfd_deactivate(struct virqfd *virqfd)
	39	{
	40	queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown);
	41	}
	42
ac6424b9	43	static int virqfd_wakeup(wait_queue_entry_t wait, unsigned mode, int sync, void key)
7e992d69 AM	44	{
7e992d69 AM	45	struct virqfd *virqfd = container_of(wait, struct virqfd, wait);
3ad6f93e	46	__poll_t flags = key_to_poll(key);
7e992d69	47
a9a08845	48	if (flags & EPOLLIN) {
b1b397ae DW	49	u64 cnt;
	50	eventfd_ctx_do_read(virqfd->eventfd, &cnt);
	51
7e992d69 AM	52	/* An event has been signaled, call function */
	53	if ((!virqfd->handler \|\|
	54	virqfd->handler(virqfd->opaque, virqfd->data)) &&
	55	virqfd->thread)
	56	schedule_work(&virqfd->inject);
	57	}
	58
a9a08845	59	if (flags & EPOLLHUP) {
7e992d69 AM	60	unsigned long flags;
	61	spin_lock_irqsave(&virqfd_lock, flags);
	62
	63	/*
	64	* The eventfd is closing, if the virqfd has not yet been
	65	* queued for release, as determined by testing whether the
	66	* virqfd pointer to it is still valid, queue it now. As
	67	* with kvm irqfds, we know we won't race against the virqfd
	68	* going away because we hold the lock to get here.
	69	*/
	70	if (*(virqfd->pvirqfd) == virqfd) {
	71	*(virqfd->pvirqfd) = NULL;
	72	virqfd_deactivate(virqfd);
	73	}
	74
	75	spin_unlock_irqrestore(&virqfd_lock, flags);
	76	}
	77
	78	return 0;
	79	}
	80
	81	static void virqfd_ptable_queue_proc(struct file *file,
	82	wait_queue_head_t wqh, poll_table pt)
	83	{
	84	struct virqfd *virqfd = container_of(pt, struct virqfd, pt);
	85	add_wait_queue(wqh, &virqfd->wait);
	86	}
	87
	88	static void virqfd_shutdown(struct work_struct *work)
	89	{
	90	struct virqfd *virqfd = container_of(work, struct virqfd, shutdown);
	91	u64 cnt;
	92
	93	eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt);
	94	flush_work(&virqfd->inject);
	95	eventfd_ctx_put(virqfd->eventfd);
	96
	97	kfree(virqfd);
	98	}
	99
	100	static void virqfd_inject(struct work_struct *work)
	101	{
	102	struct virqfd *virqfd = container_of(work, struct virqfd, inject);
	103	if (virqfd->thread)
	104	virqfd->thread(virqfd->opaque, virqfd->data);
	105	}
	106
	107	int vfio_virqfd_enable(void *opaque,
	108	int (handler)(void , void *),
	109	void (thread)(void , void *),
	110	void data, struct virqfd *pvirqfd, int fd)
	111	{
	112	struct fd irqfd;
	113	struct eventfd_ctx *ctx;
	114	struct virqfd *virqfd;
	115	int ret = 0;
e6c8adca	116	__poll_t events;
7e992d69 AM	117
	118	virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL);
	119	if (!virqfd)
	120	return -ENOMEM;
	121
	122	virqfd->pvirqfd = pvirqfd;
	123	virqfd->opaque = opaque;
	124	virqfd->handler = handler;
	125	virqfd->thread = thread;
	126	virqfd->data = data;
	127
	128	INIT_WORK(&virqfd->shutdown, virqfd_shutdown);
	129	INIT_WORK(&virqfd->inject, virqfd_inject);
	130
	131	irqfd = fdget(fd);
	132	if (!irqfd.file) {
	133	ret = -EBADF;
	134	goto err_fd;
	135	}
	136
	137	ctx = eventfd_ctx_fileget(irqfd.file);
	138	if (IS_ERR(ctx)) {
	139	ret = PTR_ERR(ctx);
	140	goto err_ctx;
	141	}
	142
	143	virqfd->eventfd = ctx;
	144
	145	/*
	146	* virqfds can be released by closing the eventfd or directly
	147	* through ioctl. These are both done through a workqueue, so
	148	* we update the pointer to the virqfd under lock to avoid
	149	* pushing multiple jobs to release the same virqfd.
	150	*/
	151	spin_lock_irq(&virqfd_lock);
	152
	153	if (*pvirqfd) {
	154	spin_unlock_irq(&virqfd_lock);
	155	ret = -EBUSY;
	156	goto err_busy;
	157	}
	158	*pvirqfd = virqfd;
	159
	160	spin_unlock_irq(&virqfd_lock);
	161
	162	/*
	163	* Install our own custom wake-up handling so we are notified via
	164	* a callback whenever someone signals the underlying eventfd.
	165	*/
	166	init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup);
	167	init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc);
	168
9965ed17	169	events = vfs_poll(irqfd.file, &virqfd->pt);
7e992d69 AM	170
	171	/*
	172	* Check if there was an event already pending on the eventfd
	173	* before we registered and trigger it as if we didn't miss it.
	174	*/
a9a08845	175	if (events & EPOLLIN) {
7e992d69 AM	176	if ((!handler \|\| handler(opaque, data)) && thread)
	177	schedule_work(&virqfd->inject);
	178	}
	179
	180	/*
	181	* Do not drop the file until the irqfd is fully initialized,
a9a08845	182	* otherwise we might race against the EPOLLHUP.
7e992d69 AM	183	*/
	184	fdput(irqfd);
	185
	186	return 0;
	187	err_busy:
	188	eventfd_ctx_put(ctx);
	189	err_ctx:
	190	fdput(irqfd);
	191	err_fd:
	192	kfree(virqfd);
	193
	194	return ret;
	195	}
	196	EXPORT_SYMBOL_GPL(vfio_virqfd_enable);
	197
	198	void vfio_virqfd_disable(struct virqfd **pvirqfd)
	199	{
	200	unsigned long flags;
	201
	202	spin_lock_irqsave(&virqfd_lock, flags);
	203
	204	if (*pvirqfd) {
	205	virqfd_deactivate(*pvirqfd);
	206	*pvirqfd = NULL;
	207	}
	208
	209	spin_unlock_irqrestore(&virqfd_lock, flags);
	210
	211	/*
	212	* Block until we know all outstanding shutdown jobs have completed.
	213	* Even if we don't queue the job, flush the wq to be sure it's
	214	* been released.
	215	*/
	216	flush_workqueue(vfio_irqfd_cleanup_wq);
	217	}
	218	EXPORT_SYMBOL_GPL(vfio_virqfd_disable);
71be3423 AW	219
	220	module_init(vfio_virqfd_init);
	221	module_exit(vfio_virqfd_exit);
	222
	223	MODULE_VERSION(DRIVER_VERSION);
	224	MODULE_LICENSE("GPL v2");
	225	MODULE_AUTHOR(DRIVER_AUTHOR);
	226	MODULE_DESCRIPTION(DRIVER_DESC);