[linux-block.git] / drivers / virtio / virtio.c

// SPDX-License-Identifier: GPL-2.0-only
#include <linux/virtio.h>
#include <linux/spinlock.h>
#include <linux/virtio_config.h>
#include <linux/virtio_anchor.h>
#include <linux/module.h>
#include <linux/idr.h>
#include <linux/of.h>
#include <uapi/linux/virtio_ids.h>

/* Unique numbering for virtio devices. */
static DEFINE_IDA(virtio_index_ida);

static ssize_t device_show(struct device *_d,
			   struct device_attribute *attr, char *buf)
{
	struct virtio_device *dev = dev_to_virtio(_d);
	return sprintf(buf, "0x%04x\n", dev->id.device);
}
static DEVICE_ATTR_RO(device);

static ssize_t vendor_show(struct device *_d,
			   struct device_attribute *attr, char *buf)
{
	struct virtio_device *dev = dev_to_virtio(_d);
	return sprintf(buf, "0x%04x\n", dev->id.vendor);
}
static DEVICE_ATTR_RO(vendor);

static ssize_t status_show(struct device *_d,
			   struct device_attribute *attr, char *buf)
{
	struct virtio_device *dev = dev_to_virtio(_d);
	return sprintf(buf, "0x%08x\n", dev->config->get_status(dev));
}
static DEVICE_ATTR_RO(status);

static ssize_t modalias_show(struct device *_d,
			     struct device_attribute *attr, char *buf)
{
	struct virtio_device *dev = dev_to_virtio(_d);
	return sprintf(buf, "virtio:d%08Xv%08X\n",
		       dev->id.device, dev->id.vendor);
}
static DEVICE_ATTR_RO(modalias);

static ssize_t features_show(struct device *_d,
			     struct device_attribute *attr, char *buf)
{
	struct virtio_device *dev = dev_to_virtio(_d);
	unsigned int i;
	ssize_t len = 0;

	/* We actually represent this as a bitstring, as it could be
	 * arbitrary length in future. */
	for (i = 0; i < sizeof(dev->features)*8; i++)
		len += sprintf(buf+len, "%c",
			       __virtio_test_bit(dev, i) ? '1' : '0');
	len += sprintf(buf+len, "\n");
	return len;
}
static DEVICE_ATTR_RO(features);

static struct attribute *virtio_dev_attrs[] = {
	&dev_attr_device.attr,
	&dev_attr_vendor.attr,
	&dev_attr_status.attr,
	&dev_attr_modalias.attr,
	&dev_attr_features.attr,
	NULL,
};
ATTRIBUTE_GROUPS(virtio_dev);

static inline int virtio_id_match(const struct virtio_device *dev,
				  const struct virtio_device_id *id)
{
	if (id->device != dev->id.device && id->device != VIRTIO_DEV_ANY_ID)
		return 0;

	return id->vendor == VIRTIO_DEV_ANY_ID || id->vendor == dev->id.vendor;
}

/* This looks through all the IDs a driver claims to support.  If any of them
 * match, we return 1 and the kernel will call virtio_dev_probe(). */
static int virtio_dev_match(struct device *_dv, struct device_driver *_dr)
{
	unsigned int i;
	struct virtio_device *dev = dev_to_virtio(_dv);
	const struct virtio_device_id *ids;

	ids = drv_to_virtio(_dr)->id_table;
	for (i = 0; ids[i].device; i++)
		if (virtio_id_match(dev, &ids[i]))
			return 1;
	return 0;
}

static int virtio_uevent(struct device *_dv, struct kobj_uevent_env *env)
{
	struct virtio_device *dev = dev_to_virtio(_dv);

	return add_uevent_var(env, "MODALIAS=virtio:d%08Xv%08X",
			      dev->id.device, dev->id.vendor);
}

void virtio_check_driver_offered_feature(const struct virtio_device *vdev,
					 unsigned int fbit)
{
	unsigned int i;
	struct virtio_driver *drv = drv_to_virtio(vdev->dev.driver);

	for (i = 0; i < drv->feature_table_size; i++)
		if (drv->feature_table[i] == fbit)
			return;

	if (drv->feature_table_legacy) {
		for (i = 0; i < drv->feature_table_size_legacy; i++)
			if (drv->feature_table_legacy[i] == fbit)
				return;
	}

	BUG();
}
EXPORT_SYMBOL_GPL(virtio_check_driver_offered_feature);

static void __virtio_config_changed(struct virtio_device *dev)
{
	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);

	if (!dev->config_enabled)
		dev->config_change_pending = true;
	else if (drv && drv->config_changed)
		drv->config_changed(dev);
}

void virtio_config_changed(struct virtio_device *dev)
{
	unsigned long flags;

	spin_lock_irqsave(&dev->config_lock, flags);
	__virtio_config_changed(dev);
	spin_unlock_irqrestore(&dev->config_lock, flags);
}
EXPORT_SYMBOL_GPL(virtio_config_changed);

static void virtio_config_disable(struct virtio_device *dev)
{
	spin_lock_irq(&dev->config_lock);
	dev->config_enabled = false;
	spin_unlock_irq(&dev->config_lock);
}

static void virtio_config_enable(struct virtio_device *dev)
{
	spin_lock_irq(&dev->config_lock);
	dev->config_enabled = true;
	if (dev->config_change_pending)
		__virtio_config_changed(dev);
	dev->config_change_pending = false;
	spin_unlock_irq(&dev->config_lock);
}

void virtio_add_status(struct virtio_device *dev, unsigned int status)
{
	might_sleep();
	dev->config->set_status(dev, dev->config->get_status(dev) | status);
}
EXPORT_SYMBOL_GPL(virtio_add_status);

/* Do some validation, then set FEATURES_OK */
static int virtio_features_ok(struct virtio_device *dev)
{
	unsigned int status;

	might_sleep();

	if (virtio_check_mem_acc_cb(dev)) {
		if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1)) {
			dev_warn(&dev->dev,
				 "device must provide VIRTIO_F_VERSION_1\n");
			return -ENODEV;
		}

		if (!virtio_has_feature(dev, VIRTIO_F_ACCESS_PLATFORM)) {
			dev_warn(&dev->dev,
				 "device must provide VIRTIO_F_ACCESS_PLATFORM\n");
			return -ENODEV;
		}
	}

	if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1))
		return 0;

	virtio_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
	status = dev->config->get_status(dev);
	if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {
		dev_err(&dev->dev, "virtio: device refuses features: %x\n",
			status);
		return -ENODEV;
	}
	return 0;
}

/**
 * virtio_reset_device - quiesce device for removal
 * @dev: the device to reset
 *
 * Prevents device from sending interrupts and accessing memory.
 *
 * Generally used for cleanup during driver / device removal.
 *
 * Once this has been invoked, caller must ensure that
 * virtqueue_notify / virtqueue_kick are not in progress.
 *
 * Note: this guarantees that vq callbacks are not in progress, however caller
 * is responsible for preventing access from other contexts, such as a system
 * call/workqueue/bh.  Invoking virtio_break_device then flushing any such
 * contexts is one way to handle that.
 * */
void virtio_reset_device(struct virtio_device *dev)
{
#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
	/*
	 * The below virtio_synchronize_cbs() guarantees that any
	 * interrupt for this line arriving after
	 * virtio_synchronize_vqs() has completed is guaranteed to see
	 * vq->broken as true.
	 */
	virtio_break_device(dev);
	virtio_synchronize_cbs(dev);
#endif

	dev->config->reset(dev);
}
EXPORT_SYMBOL_GPL(virtio_reset_device);

static int virtio_dev_probe(struct device *_d)
{
	int err, i;
	struct virtio_device *dev = dev_to_virtio(_d);
	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
	u64 device_features;
	u64 driver_features;
	u64 driver_features_legacy;

	/* We have a driver! */
	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);

	/* Figure out what features the device supports. */
	device_features = dev->config->get_features(dev);

	/* Figure out what features the driver supports. */
	driver_features = 0;
	for (i = 0; i < drv->feature_table_size; i++) {
		unsigned int f = drv->feature_table[i];
		BUG_ON(f >= 64);
		driver_features |= (1ULL << f);
	}

	/* Some drivers have a separate feature table for virtio v1.0 */
	if (drv->feature_table_legacy) {
		driver_features_legacy = 0;
		for (i = 0; i < drv->feature_table_size_legacy; i++) {
			unsigned int f = drv->feature_table_legacy[i];
			BUG_ON(f >= 64);
			driver_features_legacy |= (1ULL << f);
		}
	} else {
		driver_features_legacy = driver_features;
	}

	if (device_features & (1ULL << VIRTIO_F_VERSION_1))
		dev->features = driver_features & device_features;
	else
		dev->features = driver_features_legacy & device_features;

	/* Transport features always preserved to pass to finalize_features. */
	for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++)
		if (device_features & (1ULL << i))
			__virtio_set_bit(dev, i);

	err = dev->config->finalize_features(dev);
	if (err)
		goto err;

	if (drv->validate) {
		u64 features = dev->features;

		err = drv->validate(dev);
		if (err)
			goto err;

		/* Did validation change any features? Then write them again. */
		if (features != dev->features) {
			err = dev->config->finalize_features(dev);
			if (err)
				goto err;
		}
	}

	err = virtio_features_ok(dev);
	if (err)
		goto err;

	err = drv->probe(dev);
	if (err)
		goto err;

	/* If probe didn't do it, mark device DRIVER_OK ourselves. */
	if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
		virtio_device_ready(dev);

	if (drv->scan)
		drv->scan(dev);

	virtio_config_enable(dev);

	return 0;
err:
	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
	return err;

}

static void virtio_dev_remove(struct device *_d)
{
	struct virtio_device *dev = dev_to_virtio(_d);
	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);

	virtio_config_disable(dev);

	drv->remove(dev);

	/* Driver should have reset device. */
	WARN_ON_ONCE(dev->config->get_status(dev));

	/* Acknowledge the device's existence again. */
	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);

	of_node_put(dev->dev.of_node);
}

static struct bus_type virtio_bus = {
	.name  = "virtio",
	.match = virtio_dev_match,
	.dev_groups = virtio_dev_groups,
	.uevent = virtio_uevent,
	.probe = virtio_dev_probe,
	.remove = virtio_dev_remove,
};

int register_virtio_driver(struct virtio_driver *driver)
{
	/* Catch this early. */
	BUG_ON(driver->feature_table_size && !driver->feature_table);
	driver->driver.bus = &virtio_bus;
	return driver_register(&driver->driver);
}
EXPORT_SYMBOL_GPL(register_virtio_driver);

void unregister_virtio_driver(struct virtio_driver *driver)
{
	driver_unregister(&driver->driver);
}
EXPORT_SYMBOL_GPL(unregister_virtio_driver);

static int virtio_device_of_init(struct virtio_device *dev)
{
	struct device_node *np, *pnode = dev_of_node(dev->dev.parent);
	char compat[] = "virtio,deviceXXXXXXXX";
	int ret, count;

	if (!pnode)
		return 0;

	count = of_get_available_child_count(pnode);
	if (!count)
		return 0;

	/* There can be only 1 child node */
	if (WARN_ON(count > 1))
		return -EINVAL;

	np = of_get_next_available_child(pnode, NULL);
	if (WARN_ON(!np))
		return -ENODEV;

	ret = snprintf(compat, sizeof(compat), "virtio,device%x", dev->id.device);
	BUG_ON(ret >= sizeof(compat));

	/*
	 * On powerpc/pseries virtio devices are PCI devices so PCI
	 * vendor/device ids play the role of the "compatible" property.
	 * Simply don't init of_node in this case.
	 */
	if (!of_device_is_compatible(np, compat)) {
		ret = 0;
		goto out;
	}

	dev->dev.of_node = np;
	return 0;

out:
	of_node_put(np);
	return ret;
}

/**
 * register_virtio_device - register virtio device
 * @dev        : virtio device to be registered
 *
 * On error, the caller must call put_device on &@dev->dev (and not kfree),
 * as another code path may have obtained a reference to @dev.
 *
 * Returns: 0 on suceess, -error on failure
 */
int register_virtio_device(struct virtio_device *dev)
{
	int err;

	dev->dev.bus = &virtio_bus;
	device_initialize(&dev->dev);

	/* Assign a unique device index and hence name. */
	err = ida_alloc(&virtio_index_ida, GFP_KERNEL);
	if (err < 0)
		goto out;

	dev->index = err;
	dev_set_name(&dev->dev, "virtio%u", dev->index);

	err = virtio_device_of_init(dev);
	if (err)
		goto out_ida_remove;

	spin_lock_init(&dev->config_lock);
	dev->config_enabled = false;
	dev->config_change_pending = false;

	INIT_LIST_HEAD(&dev->vqs);
	spin_lock_init(&dev->vqs_list_lock);

	/* We always start by resetting the device, in case a previous
	 * driver messed it up.  This also tests that code path a little. */
	virtio_reset_device(dev);

	/* Acknowledge that we've seen the device. */
	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);

	/*
	 * device_add() causes the bus infrastructure to look for a matching
	 * driver.
	 */
	err = device_add(&dev->dev);
	if (err)
		goto out_of_node_put;

	return 0;

out_of_node_put:
	of_node_put(dev->dev.of_node);
out_ida_remove:
	ida_free(&virtio_index_ida, dev->index);
out:
	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
	return err;
}
EXPORT_SYMBOL_GPL(register_virtio_device);

bool is_virtio_device(struct device *dev)
{
	return dev->bus == &virtio_bus;
}
EXPORT_SYMBOL_GPL(is_virtio_device);

void unregister_virtio_device(struct virtio_device *dev)
{
	int index = dev->index; /* save for after device release */

	device_unregister(&dev->dev);
	ida_free(&virtio_index_ida, index);
}
EXPORT_SYMBOL_GPL(unregister_virtio_device);

#ifdef CONFIG_PM_SLEEP
int virtio_device_freeze(struct virtio_device *dev)
{
	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);

	virtio_config_disable(dev);

	dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;

	if (drv && drv->freeze)
		return drv->freeze(dev);

	return 0;
}
EXPORT_SYMBOL_GPL(virtio_device_freeze);

int virtio_device_restore(struct virtio_device *dev)
{
	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
	int ret;

	/* We always start by resetting the device, in case a previous
	 * driver messed it up. */
	virtio_reset_device(dev);

	/* Acknowledge that we've seen the device. */
	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);

	/* Maybe driver failed before freeze.
	 * Restore the failed status, for debugging. */
	if (dev->failed)
		virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);

	if (!drv)
		return 0;

	/* We have a driver! */
	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);

	ret = dev->config->finalize_features(dev);
	if (ret)
		goto err;

	ret = virtio_features_ok(dev);
	if (ret)
		goto err;

	if (drv->restore) {
		ret = drv->restore(dev);
		if (ret)
			goto err;
	}

	/* If restore didn't do it, mark device DRIVER_OK ourselves. */
	if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
		virtio_device_ready(dev);

	virtio_config_enable(dev);

	return 0;

err:
	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
	return ret;
}
EXPORT_SYMBOL_GPL(virtio_device_restore);
#endif

static int virtio_init(void)
{
	if (bus_register(&virtio_bus) != 0)
		panic("virtio bus registration failed");
	return 0;
}

static void __exit virtio_exit(void)
{
	bus_unregister(&virtio_bus);
	ida_destroy(&virtio_index_ida);
}
core_initcall(virtio_init);
module_exit(virtio_exit);

MODULE_LICENSE("GPL");
Commit	Line	Data
09c434b8	1	// SPDX-License-Identifier: GPL-2.0-only
ec3d41c4 RR	2	#include <linux/virtio.h>
	3	#include <linux/spinlock.h>
	4	#include <linux/virtio_config.h>
a603002e	5	#include <linux/virtio_anchor.h>
b5a2c4f1	6	#include <linux/module.h>
90e03207	7	#include <linux/idr.h>
694a1116	8	#include <linux/of.h>
b6098c30	9	#include <uapi/linux/virtio_ids.h>
ec3d41c4	10
b769f579	11	/* Unique numbering for virtio devices. */
90e03207	12	static DEFINE_IDA(virtio_index_ida);
b769f579	13
ec3d41c4 RR	14	static ssize_t device_show(struct device *_d,
	15	struct device_attribute attr, char buf)
	16	{
9bffdca8	17	struct virtio_device *dev = dev_to_virtio(_d);
be6528b2	18	return sprintf(buf, "0x%04x\n", dev->id.device);
ec3d41c4	19	}
3736dab6 GKH	20	static DEVICE_ATTR_RO(device);
3736dab6 GKH	21
ec3d41c4 RR	22	static ssize_t vendor_show(struct device *_d,
	23	struct device_attribute attr, char buf)
	24	{
9bffdca8	25	struct virtio_device *dev = dev_to_virtio(_d);
be6528b2	26	return sprintf(buf, "0x%04x\n", dev->id.vendor);
ec3d41c4	27	}
3736dab6 GKH	28	static DEVICE_ATTR_RO(vendor);
3736dab6 GKH	29
ec3d41c4 RR	30	static ssize_t status_show(struct device *_d,
	31	struct device_attribute attr, char buf)
	32	{
9bffdca8	33	struct virtio_device *dev = dev_to_virtio(_d);
be6528b2	34	return sprintf(buf, "0x%08x\n", dev->config->get_status(dev));
ec3d41c4	35	}
3736dab6 GKH	36	static DEVICE_ATTR_RO(status);
3736dab6 GKH	37
b01d9f28 RR	38	static ssize_t modalias_show(struct device *_d,
	39	struct device_attribute attr, char buf)
	40	{
9bffdca8	41	struct virtio_device *dev = dev_to_virtio(_d);
b01d9f28 RR	42	return sprintf(buf, "virtio:d%08Xv%08X\n",
	43	dev->id.device, dev->id.vendor);
	44	}
3736dab6 GKH	45	static DEVICE_ATTR_RO(modalias);
3736dab6 GKH	46
a9289282 RR	47	static ssize_t features_show(struct device *_d,
	48	struct device_attribute attr, char buf)
	49	{
9bffdca8	50	struct virtio_device *dev = dev_to_virtio(_d);
a9289282 RR	51	unsigned int i;
	52	ssize_t len = 0;
	53
	54	/* We actually represent this as a bitstring, as it could be
	55	* arbitrary length in future. */
e16e12be	56	for (i = 0; i < sizeof(dev->features)*8; i++)
a9289282	57	len += sprintf(buf+len, "%c",
e16e12be	58	__virtio_test_bit(dev, i) ? '1' : '0');
a9289282 RR	59	len += sprintf(buf+len, "\n");
	60	return len;
	61	}
3736dab6 GKH	62	static DEVICE_ATTR_RO(features);
	63
	64	static struct attribute *virtio_dev_attrs[] = {
	65	&dev_attr_device.attr,
	66	&dev_attr_vendor.attr,
	67	&dev_attr_status.attr,
	68	&dev_attr_modalias.attr,
	69	&dev_attr_features.attr,
	70	NULL,
ec3d41c4	71	};
3736dab6	72	ATTRIBUTE_GROUPS(virtio_dev);
ec3d41c4 RR	73
	74	static inline int virtio_id_match(const struct virtio_device *dev,
	75	const struct virtio_device_id *id)
	76	{
e3353853	77	if (id->device != dev->id.device && id->device != VIRTIO_DEV_ANY_ID)
ec3d41c4 RR	78	return 0;
ec3d41c4 RR	79
c89e8016	80	return id->vendor == VIRTIO_DEV_ANY_ID \|\| id->vendor == dev->id.vendor;
ec3d41c4 RR	81	}
	82
	83	/* This looks through all the IDs a driver claims to support. If any of them
	84	* match, we return 1 and the kernel will call virtio_dev_probe(). */
	85	static int virtio_dev_match(struct device _dv, struct device_driver _dr)
	86	{
	87	unsigned int i;
9bffdca8	88	struct virtio_device *dev = dev_to_virtio(_dv);
ec3d41c4 RR	89	const struct virtio_device_id *ids;
ec3d41c4 RR	90
9a2bdcc8	91	ids = drv_to_virtio(_dr)->id_table;
ec3d41c4 RR	92	for (i = 0; ids[i].device; i++)
	93	if (virtio_id_match(dev, &ids[i]))
	94	return 1;
	95	return 0;
	96	}
	97
b01d9f28 RR	98	static int virtio_uevent(struct device _dv, struct kobj_uevent_env env)
b01d9f28 RR	99	{
9bffdca8	100	struct virtio_device *dev = dev_to_virtio(_dv);
b01d9f28 RR	101
	102	return add_uevent_var(env, "MODALIAS=virtio:d%08Xv%08X",
	103	dev->id.device, dev->id.vendor);
	104	}
	105
c45a6816 RR	106	void virtio_check_driver_offered_feature(const struct virtio_device *vdev,
	107	unsigned int fbit)
	108	{
	109	unsigned int i;
9a2bdcc8	110	struct virtio_driver *drv = drv_to_virtio(vdev->dev.driver);
c45a6816 RR	111
	112	for (i = 0; i < drv->feature_table_size; i++)
	113	if (drv->feature_table[i] == fbit)
	114	return;
b3bb62d1 MT	115
	116	if (drv->feature_table_legacy) {
	117	for (i = 0; i < drv->feature_table_size_legacy; i++)
	118	if (drv->feature_table_legacy[i] == fbit)
	119	return;
	120	}
	121
c45a6816 RR	122	BUG();
	123	}
	124	EXPORT_SYMBOL_GPL(virtio_check_driver_offered_feature);
	125
22b7050a MT	126	static void __virtio_config_changed(struct virtio_device *dev)
	127	{
	128	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
	129
	130	if (!dev->config_enabled)
	131	dev->config_change_pending = true;
	132	else if (drv && drv->config_changed)
	133	drv->config_changed(dev);
	134	}
	135
	136	void virtio_config_changed(struct virtio_device *dev)
	137	{
	138	unsigned long flags;
	139
	140	spin_lock_irqsave(&dev->config_lock, flags);
	141	__virtio_config_changed(dev);
	142	spin_unlock_irqrestore(&dev->config_lock, flags);
	143	}
	144	EXPORT_SYMBOL_GPL(virtio_config_changed);
	145
bc22ed2e	146	static void virtio_config_disable(struct virtio_device *dev)
22b7050a MT	147	{
	148	spin_lock_irq(&dev->config_lock);
	149	dev->config_enabled = false;
	150	spin_unlock_irq(&dev->config_lock);
	151	}
	152
bc22ed2e	153	static void virtio_config_enable(struct virtio_device *dev)
22b7050a MT	154	{
	155	spin_lock_irq(&dev->config_lock);
	156	dev->config_enabled = true;
	157	if (dev->config_change_pending)
	158	__virtio_config_changed(dev);
	159	dev->config_change_pending = false;
	160	spin_unlock_irq(&dev->config_lock);
	161	}
9fe7bfce JF	162
	163	void virtio_add_status(struct virtio_device *dev, unsigned int status)
	164	{
ab7a2375	165	might_sleep();
9fe7bfce JF	166	dev->config->set_status(dev, dev->config->get_status(dev) \| status);
	167	}
	168	EXPORT_SYMBOL_GPL(virtio_add_status);
22b7050a	169
4fa59ede MT	170	/* Do some validation, then set FEATURES_OK */
4fa59ede MT	171	static int virtio_features_ok(struct virtio_device *dev)
30683a8c	172	{
31532340	173	unsigned int status;
30683a8c	174
ab7a2375	175	might_sleep();
30683a8c	176
a603002e	177	if (virtio_check_mem_acc_cb(dev)) {
0afa15e1 PM	178	if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1)) {
	179	dev_warn(&dev->dev,
	180	"device must provide VIRTIO_F_VERSION_1\n");
	181	return -ENODEV;
	182	}
	183
	184	if (!virtio_has_feature(dev, VIRTIO_F_ACCESS_PLATFORM)) {
	185	dev_warn(&dev->dev,
	186	"device must provide VIRTIO_F_ACCESS_PLATFORM\n");
	187	return -ENODEV;
	188	}
	189	}
	190
30683a8c MT	191	if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1))
	192	return 0;
	193
9fe7bfce	194	virtio_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
30683a8c MT	195	status = dev->config->get_status(dev);
	196	if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {
	197	dev_err(&dev->dev, "virtio: device refuses features: %x\n",
	198	status);
	199	return -ENODEV;
	200	}
	201	return 0;
	202	}
	203
c46eccda MT	204	/**
	205	* virtio_reset_device - quiesce device for removal
	206	* @dev: the device to reset
	207	*
	208	* Prevents device from sending interrupts and accessing memory.
	209	*
	210	* Generally used for cleanup during driver / device removal.
	211	*
	212	* Once this has been invoked, caller must ensure that
	213	* virtqueue_notify / virtqueue_kick are not in progress.
	214	*
	215	* Note: this guarantees that vq callbacks are not in progress, however caller
	216	* is responsible for preventing access from other contexts, such as a system
	217	* call/workqueue/bh. Invoking virtio_break_device then flushing any such
	218	* contexts is one way to handle that.
	219	* */
d9679d00 MT	220	void virtio_reset_device(struct virtio_device *dev)
d9679d00 MT	221	{
c346dae4	222	#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
8b4ec69d JW	223	/*
	224	* The below virtio_synchronize_cbs() guarantees that any
	225	* interrupt for this line arriving after
	226	* virtio_synchronize_vqs() has completed is guaranteed to see
	227	* vq->broken as true.
	228	*/
	229	virtio_break_device(dev);
	230	virtio_synchronize_cbs(dev);
c346dae4	231	#endif
8b4ec69d	232
d9679d00 MT	233	dev->config->reset(dev);
	234	}
	235	EXPORT_SYMBOL_GPL(virtio_reset_device);
	236
ec3d41c4 RR	237	static int virtio_dev_probe(struct device *_d)
ec3d41c4 RR	238	{
c45a6816	239	int err, i;
9bffdca8	240	struct virtio_device *dev = dev_to_virtio(_d);
9a2bdcc8	241	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
d0254773	242	u64 device_features;
c102659d	243	u64 driver_features;
b3bb62d1	244	u64 driver_features_legacy;
ec3d41c4	245
c45a6816	246	/* We have a driver! */
9fe7bfce	247	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);
c45a6816 RR	248
	249	/* Figure out what features the device supports. */
	250	device_features = dev->config->get_features(dev);
	251
c102659d MT	252	/* Figure out what features the driver supports. */
c102659d MT	253	driver_features = 0;
c45a6816 RR	254	for (i = 0; i < drv->feature_table_size; i++) {
c45a6816 RR	255	unsigned int f = drv->feature_table[i];
d0254773	256	BUG_ON(f >= 64);
c102659d	257	driver_features \|= (1ULL << f);
c45a6816 RR	258	}
c45a6816 RR	259
b3bb62d1 MT	260	/* Some drivers have a separate feature table for virtio v1.0 */
	261	if (drv->feature_table_legacy) {
	262	driver_features_legacy = 0;
	263	for (i = 0; i < drv->feature_table_size_legacy; i++) {
	264	unsigned int f = drv->feature_table_legacy[i];
	265	BUG_ON(f >= 64);
	266	driver_features_legacy \|= (1ULL << f);
	267	}
	268	} else {
	269	driver_features_legacy = driver_features;
	270	}
	271
747ae34a	272	if (device_features & (1ULL << VIRTIO_F_VERSION_1))
b3bb62d1 MT	273	dev->features = driver_features & device_features;
	274	else
	275	dev->features = driver_features_legacy & device_features;
c102659d	276
c624896e	277	/* Transport features always preserved to pass to finalize_features. */
dd7c7bc4	278	for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++)
d0254773	279	if (device_features & (1ULL << i))
e16e12be	280	__virtio_set_bit(dev, i);
dd7c7bc4	281
4fa59ede MT	282	err = dev->config->finalize_features(dev);
	283	if (err)
	284	goto err;
	285
404123c2	286	if (drv->validate) {
4fa59ede MT	287	u64 features = dev->features;
4fa59ede MT	288
404123c2 MT	289	err = drv->validate(dev);
	290	if (err)
	291	goto err;
4fa59ede MT	292
	293	/* Did validation change any features? Then write them again. */
	294	if (features != dev->features) {
	295	err = dev->config->finalize_features(dev);
	296	if (err)
	297	goto err;
	298	}
404123c2 MT	299	}
404123c2 MT	300
4fa59ede	301	err = virtio_features_ok(dev);
5c609a5e MT	302	if (err)
5c609a5e MT	303	goto err;
ef688e15	304
ec3d41c4 RR	305	err = drv->probe(dev);
ec3d41c4 RR	306	if (err)
cb3f6d9d	307	goto err;
22b7050a	308
5b40a7da RR	309	/* If probe didn't do it, mark device DRIVER_OK ourselves. */
	310	if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
	311	virtio_device_ready(dev);
	312
cb3f6d9d MT	313	if (drv->scan)
cb3f6d9d MT	314	drv->scan(dev);
ef688e15	315
cb3f6d9d MT	316	virtio_config_enable(dev);
	317
	318	return 0;
	319	err:
9fe7bfce	320	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
ec3d41c4	321	return err;
cb3f6d9d	322
ec3d41c4 RR	323	}
ec3d41c4 RR	324
fc7a6209	325	static void virtio_dev_remove(struct device *_d)
74b2553f	326	{
9bffdca8	327	struct virtio_device *dev = dev_to_virtio(_d);
9a2bdcc8	328	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
74b2553f	329
22b7050a MT	330	virtio_config_disable(dev);
22b7050a MT	331
74b2553f	332	drv->remove(dev);
6e5aa7ef RR	333
6e5aa7ef RR	334	/* Driver should have reset device. */
5543a6ac	335	WARN_ON_ONCE(dev->config->get_status(dev));
6e5aa7ef RR	336
6e5aa7ef RR	337	/* Acknowledge the device's existence again. */
9fe7bfce	338	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
694a1116 VK	339
694a1116 VK	340	of_node_put(dev->dev.of_node);
74b2553f RR	341	}
74b2553f RR	342
e962fa66 MM	343	static struct bus_type virtio_bus = {
	344	.name = "virtio",
	345	.match = virtio_dev_match,
3736dab6	346	.dev_groups = virtio_dev_groups,
e962fa66 MM	347	.uevent = virtio_uevent,
	348	.probe = virtio_dev_probe,
	349	.remove = virtio_dev_remove,
	350	};
	351
ec3d41c4 RR	352	int register_virtio_driver(struct virtio_driver *driver)
ec3d41c4 RR	353	{
c45a6816 RR	354	/* Catch this early. */
c45a6816 RR	355	BUG_ON(driver->feature_table_size && !driver->feature_table);
ec3d41c4	356	driver->driver.bus = &virtio_bus;
ec3d41c4 RR	357	return driver_register(&driver->driver);
	358	}
	359	EXPORT_SYMBOL_GPL(register_virtio_driver);
	360
	361	void unregister_virtio_driver(struct virtio_driver *driver)
	362	{
	363	driver_unregister(&driver->driver);
	364	}
	365	EXPORT_SYMBOL_GPL(unregister_virtio_driver);
	366
694a1116 VK	367	static int virtio_device_of_init(struct virtio_device *dev)
	368	{
	369	struct device_node np, pnode = dev_of_node(dev->dev.parent);
	370	char compat[] = "virtio,deviceXXXXXXXX";
	371	int ret, count;
	372
	373	if (!pnode)
	374	return 0;
	375
	376	count = of_get_available_child_count(pnode);
	377	if (!count)
	378	return 0;
	379
	380	/* There can be only 1 child node */
	381	if (WARN_ON(count > 1))
	382	return -EINVAL;
	383
	384	np = of_get_next_available_child(pnode, NULL);
	385	if (WARN_ON(!np))
	386	return -ENODEV;
	387
	388	ret = snprintf(compat, sizeof(compat), "virtio,device%x", dev->id.device);
	389	BUG_ON(ret >= sizeof(compat));
	390
0d818706 MT	391	/*
	392	* On powerpc/pseries virtio devices are PCI devices so PCI
	393	* vendor/device ids play the role of the "compatible" property.
	394	* Simply don't init of_node in this case.
	395	*/
694a1116	396	if (!of_device_is_compatible(np, compat)) {
0d818706	397	ret = 0;
694a1116 VK	398	goto out;
	399	}
	400
	401	dev->dev.of_node = np;
	402	return 0;
	403
	404	out:
	405	of_node_put(np);
	406	return ret;
	407	}
	408
f2b44cde	409	/**
	410	* register_virtio_device - register virtio device
	411	* @dev : virtio device to be registered
	412	*
	413	* On error, the caller must call put_device on &@dev->dev (and not kfree),
	414	* as another code path may have obtained a reference to @dev.
	415	*
	416	* Returns: 0 on suceess, -error on failure
	417	*/
ec3d41c4 RR	418	int register_virtio_device(struct virtio_device *dev)
	419	{
	420	int err;
	421
	422	dev->dev.bus = &virtio_bus;
f2b44cde	423	device_initialize(&dev->dev);
b769f579 RR	424
b769f579 RR	425	/* Assign a unique device index and hence name. */
4f58afd6	426	err = ida_alloc(&virtio_index_ida, GFP_KERNEL);
90e03207 AH	427	if (err < 0)
	428	goto out;
	429
	430	dev->index = err;
99e0b6c8	431	dev_set_name(&dev->dev, "virtio%u", dev->index);
ec3d41c4	432
694a1116 VK	433	err = virtio_device_of_init(dev);
	434	if (err)
	435	goto out_ida_remove;
	436
22b7050a MT	437	spin_lock_init(&dev->config_lock);
	438	dev->config_enabled = false;
	439	dev->config_change_pending = false;
	440
8b4ec69d JW	441	INIT_LIST_HEAD(&dev->vqs);
	442	spin_lock_init(&dev->vqs_list_lock);
	443
6e5aa7ef RR	444	/* We always start by resetting the device, in case a previous
6e5aa7ef RR	445	* driver messed it up. This also tests that code path a little. */
0aa96837	446	virtio_reset_device(dev);
6e5aa7ef	447
ec3d41c4	448	/* Acknowledge that we've seen the device. */
9fe7bfce	449	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
ec3d41c4	450
f2b44cde	451	/*
	452	* device_add() causes the bus infrastructure to look for a matching
	453	* driver.
	454	*/
	455	err = device_add(&dev->dev);
e60ea67b	456	if (err)
694a1116 VK	457	goto out_of_node_put;
	458
	459	return 0;
	460
	461	out_of_node_put:
	462	of_node_put(dev->dev.of_node);
	463	out_ida_remove:
4f58afd6	464	ida_free(&virtio_index_ida, dev->index);
90e03207	465	out:
694a1116	466	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
ec3d41c4 RR	467	return err;
	468	}
	469	EXPORT_SYMBOL_GPL(register_virtio_device);
	470
a0308938 DS	471	bool is_virtio_device(struct device *dev)
	472	{
	473	return dev->bus == &virtio_bus;
	474	}
	475	EXPORT_SYMBOL_GPL(is_virtio_device);
	476
ec3d41c4 RR	477	void unregister_virtio_device(struct virtio_device *dev)
ec3d41c4 RR	478	{
237242bd CH	479	int index = dev->index; /* save for after device release */
237242bd CH	480
ec3d41c4	481	device_unregister(&dev->dev);
4f58afd6	482	ida_free(&virtio_index_ida, index);
ec3d41c4 RR	483	}
	484	EXPORT_SYMBOL_GPL(unregister_virtio_device);
	485
c6716bae MT	486	#ifdef CONFIG_PM_SLEEP
	487	int virtio_device_freeze(struct virtio_device *dev)
	488	{
	489	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
	490
22b7050a MT	491	virtio_config_disable(dev);
22b7050a MT	492
c6716bae MT	493	dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;
	494
	495	if (drv && drv->freeze)
	496	return drv->freeze(dev);
	497
	498	return 0;
	499	}
	500	EXPORT_SYMBOL_GPL(virtio_device_freeze);
	501
	502	int virtio_device_restore(struct virtio_device *dev)
	503	{
	504	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
5c609a5e	505	int ret;
c6716bae MT	506
	507	/* We always start by resetting the device, in case a previous
	508	* driver messed it up. */
0aa96837	509	virtio_reset_device(dev);
c6716bae MT	510
c6716bae MT	511	/* Acknowledge that we've seen the device. */
9fe7bfce	512	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
c6716bae MT	513
	514	/* Maybe driver failed before freeze.
	515	* Restore the failed status, for debugging. */
	516	if (dev->failed)
9fe7bfce	517	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
c6716bae MT	518
	519	if (!drv)
	520	return 0;
	521
	522	/* We have a driver! */
9fe7bfce	523	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);
c6716bae	524
4fa59ede MT	525	ret = dev->config->finalize_features(dev);
	526	if (ret)
	527	goto err;
	528
	529	ret = virtio_features_ok(dev);
5c609a5e MT	530	if (ret)
5c609a5e MT	531	goto err;
c6716bae MT	532
c6716bae MT	533	if (drv->restore) {
5c609a5e MT	534	ret = drv->restore(dev);
	535	if (ret)
	536	goto err;
c6716bae MT	537	}
c6716bae MT	538
2536b2ca SG	539	/* If restore didn't do it, mark device DRIVER_OK ourselves. */
	540	if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
	541	virtio_device_ready(dev);
c6716bae	542
22b7050a MT	543	virtio_config_enable(dev);
22b7050a MT	544
c6716bae	545	return 0;
5c609a5e MT	546
5c609a5e MT	547	err:
9fe7bfce	548	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
5c609a5e	549	return ret;
c6716bae MT	550	}
	551	EXPORT_SYMBOL_GPL(virtio_device_restore);
	552	#endif
	553
ec3d41c4 RR	554	static int virtio_init(void)
	555	{
	556	if (bus_register(&virtio_bus) != 0)
	557	panic("virtio bus registration failed");
	558	return 0;
	559	}
c6fd4701 RR	560
	561	static void __exit virtio_exit(void)
	562	{
	563	bus_unregister(&virtio_bus);
c13f99b7	564	ida_destroy(&virtio_index_ida);
c6fd4701	565	}
ec3d41c4	566	core_initcall(virtio_init);
c6fd4701 RR	567	module_exit(virtio_exit);
	568
	569	MODULE_LICENSE("GPL");