size_t offset;
};
+struct virtnet_sq_free_stats {
+ u64 packets;
+ u64 bytes;
+};
+
struct virtnet_sq_stats {
struct u64_stats_sync syncp;
u64_stats_t packets;
/* Work struct for config space updates */
struct work_struct config_work;
+ /* Work struct for setting rx mode */
+ struct work_struct rx_mode_work;
+
+ /* OK to queue work setting RX mode? */
+ bool rx_mode_work_enabled;
+
/* Does the affinity hint is set for virtqueues? */
bool affinity_hint_set;
return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
}
+static void __free_old_xmit(struct send_queue *sq, bool in_napi,
+ struct virtnet_sq_free_stats *stats)
+{
+ unsigned int len;
+ void *ptr;
+
+ while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
+ ++stats->packets;
+
+ if (!is_xdp_frame(ptr)) {
+ struct sk_buff *skb = ptr;
+
+ pr_debug("Sent skb %p\n", skb);
+
+ stats->bytes += skb->len;
+ napi_consume_skb(skb, in_napi);
+ } else {
+ struct xdp_frame *frame = ptr_to_xdp(ptr);
+
+ stats->bytes += xdp_get_frame_len(frame);
+ xdp_return_frame(frame);
+ }
+ }
+}
+
/* Converting between virtqueue no. and kernel tx/rx queue no.
* 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
*/
spin_unlock_bh(&vi->refill_lock);
}
+static void enable_rx_mode_work(struct virtnet_info *vi)
+{
+ rtnl_lock();
+ vi->rx_mode_work_enabled = true;
+ rtnl_unlock();
+}
+
+static void disable_rx_mode_work(struct virtnet_info *vi)
+{
+ rtnl_lock();
+ vi->rx_mode_work_enabled = false;
+ rtnl_unlock();
+}
+
static void virtqueue_napi_schedule(struct napi_struct *napi,
struct virtqueue *vq)
{
virtnet_rq_free_buf(vi, rq, buf);
}
-static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi)
+static void free_old_xmit(struct send_queue *sq, bool in_napi)
{
- unsigned int len;
- unsigned int packets = 0;
- unsigned int bytes = 0;
- void *ptr;
-
- while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
- if (likely(!is_xdp_frame(ptr))) {
- struct sk_buff *skb = ptr;
-
- pr_debug("Sent skb %p\n", skb);
+ struct virtnet_sq_free_stats stats = {0};
- bytes += skb->len;
- napi_consume_skb(skb, in_napi);
- } else {
- struct xdp_frame *frame = ptr_to_xdp(ptr);
-
- bytes += xdp_get_frame_len(frame);
- xdp_return_frame(frame);
- }
- packets++;
- }
+ __free_old_xmit(sq, in_napi, &stats);
/* Avoid overhead when no packets have been processed
* happens when called speculatively from start_xmit.
*/
- if (!packets)
+ if (!stats.packets)
return;
u64_stats_update_begin(&sq->stats.syncp);
- u64_stats_add(&sq->stats.bytes, bytes);
- u64_stats_add(&sq->stats.packets, packets);
+ u64_stats_add(&sq->stats.bytes, stats.bytes);
+ u64_stats_add(&sq->stats.packets, stats.packets);
u64_stats_update_end(&sq->stats.syncp);
}
virtqueue_napi_schedule(&sq->napi, sq->vq);
} else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
/* More just got used, free them then recheck. */
- free_old_xmit_skbs(sq, false);
+ free_old_xmit(sq, false);
if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
netif_start_subqueue(dev, qnum);
virtqueue_disable_cb(sq->vq);
int n, struct xdp_frame **frames, u32 flags)
{
struct virtnet_info *vi = netdev_priv(dev);
+ struct virtnet_sq_free_stats stats = {0};
struct receive_queue *rq = vi->rq;
struct bpf_prog *xdp_prog;
struct send_queue *sq;
- unsigned int len;
- int packets = 0;
- int bytes = 0;
int nxmit = 0;
int kicks = 0;
- void *ptr;
int ret;
int i;
}
/* Free up any pending old buffers before queueing new ones. */
- while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
- if (likely(is_xdp_frame(ptr))) {
- struct xdp_frame *frame = ptr_to_xdp(ptr);
-
- bytes += xdp_get_frame_len(frame);
- xdp_return_frame(frame);
- } else {
- struct sk_buff *skb = ptr;
-
- bytes += skb->len;
- napi_consume_skb(skb, false);
- }
- packets++;
- }
+ __free_old_xmit(sq, false, &stats);
for (i = 0; i < n; i++) {
struct xdp_frame *xdpf = frames[i];
}
out:
u64_stats_update_begin(&sq->stats.syncp);
- u64_stats_add(&sq->stats.bytes, bytes);
- u64_stats_add(&sq->stats.packets, packets);
+ u64_stats_add(&sq->stats.bytes, stats.bytes);
+ u64_stats_add(&sq->stats.packets, stats.packets);
u64_stats_add(&sq->stats.xdp_tx, n);
u64_stats_add(&sq->stats.xdp_tx_drops, n - nxmit);
u64_stats_add(&sq->stats.kicks, kicks);
do {
virtqueue_disable_cb(sq->vq);
- free_old_xmit_skbs(sq, true);
+ free_old_xmit(sq, true);
} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
txq = netdev_get_tx_queue(vi->dev, index);
__netif_tx_lock(txq, raw_smp_processor_id());
virtqueue_disable_cb(sq->vq);
- free_old_xmit_skbs(sq, true);
+ free_old_xmit(sq, true);
if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
netif_tx_wake_queue(txq);
if (use_napi)
virtqueue_disable_cb(sq->vq);
- free_old_xmit_skbs(sq, false);
+ free_old_xmit(sq, false);
} while (use_napi && kick &&
unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
* into the hypervisor, so the request should be handled immediately.
*/
while (!virtqueue_get_buf(vi->cvq, &tmp) &&
- !virtqueue_is_broken(vi->cvq))
+ !virtqueue_is_broken(vi->cvq)) {
+ cond_resched();
cpu_relax();
+ }
return vi->ctrl->status == VIRTIO_NET_OK;
}
return 0;
}
-static void virtnet_set_rx_mode(struct net_device *dev)
+static void virtnet_rx_mode_work(struct work_struct *work)
{
- struct virtnet_info *vi = netdev_priv(dev);
+ struct virtnet_info *vi =
+ container_of(work, struct virtnet_info, rx_mode_work);
+ struct net_device *dev = vi->dev;
struct scatterlist sg[2];
struct virtio_net_ctrl_mac *mac_data;
struct netdev_hw_addr *ha;
if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
return;
+ rtnl_lock();
+
vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0);
vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
vi->ctrl->allmulti ? "en" : "dis");
+ netif_addr_lock_bh(dev);
+
uc_count = netdev_uc_count(dev);
mc_count = netdev_mc_count(dev);
/* MAC filter - use one buffer for both lists */
buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
(2 * sizeof(mac_data->entries)), GFP_ATOMIC);
mac_data = buf;
- if (!buf)
+ if (!buf) {
+ netif_addr_unlock_bh(dev);
+ rtnl_unlock();
return;
+ }
sg_init_table(sg, 2);
netdev_for_each_mc_addr(ha, dev)
memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
+ netif_addr_unlock_bh(dev);
+
sg_set_buf(&sg[1], mac_data,
sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
+ rtnl_unlock();
+
kfree(buf);
}
+static void virtnet_set_rx_mode(struct net_device *dev)
+{
+ struct virtnet_info *vi = netdev_priv(dev);
+
+ if (vi->rx_mode_work_enabled)
+ schedule_work(&vi->rx_mode_work);
+}
+
static int virtnet_vlan_rx_add_vid(struct net_device *dev,
__be16 proto, u16 vid)
{
/* Make sure no work handler is accessing the device */
flush_work(&vi->config_work);
+ disable_rx_mode_work(vi);
+ flush_work(&vi->rx_mode_work);
netif_tx_lock_bh(vi->dev);
netif_device_detach(vi->dev);
virtio_device_ready(vdev);
enable_delayed_refill(vi);
+ enable_rx_mode_work(vi);
if (netif_running(vi->dev)) {
err = virtnet_open(vi->dev);
vdev->priv = vi;
INIT_WORK(&vi->config_work, virtnet_config_changed_work);
+ INIT_WORK(&vi->rx_mode_work, virtnet_rx_mode_work);
spin_lock_init(&vi->refill_lock);
if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) {
if (vi->has_rss || vi->has_rss_hash_report)
virtnet_init_default_rss(vi);
+ enable_rx_mode_work(vi);
+
/* serialize netdev register + virtio_device_ready() with ndo_open() */
rtnl_lock();
/* Make sure no work handler is accessing the device. */
flush_work(&vi->config_work);
+ disable_rx_mode_work(vi);
+ flush_work(&vi->rx_mode_work);
unregister_netdev(vi->dev);
return vp_legacy_get_queue_size(ldev, 0);
}
+static u16 eni_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 qid)
+{
+ struct virtio_pci_legacy_device *ldev = vdpa_to_ldev(vdpa);
+
+ return vp_legacy_get_queue_size(ldev, qid);
+}
+
static int eni_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 qid,
struct vdpa_vq_state *state)
{
.reset = eni_vdpa_reset,
.get_vq_num_max = eni_vdpa_get_vq_num_max,
.get_vq_num_min = eni_vdpa_get_vq_num_min,
+ .get_vq_size = eni_vdpa_get_vq_size,
.get_vq_state = eni_vdpa_get_vq_state,
.set_vq_state = eni_vdpa_set_vq_state,
.set_vq_cb = eni_vdpa_set_vq_cb,
return 0;
}
-static u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid)
+u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid)
{
u16 queue_size;
+ if (qid >= hw->nr_vring)
+ return 0;
+
vp_iowrite16(qid, &hw->common_cfg->queue_select);
queue_size = vp_ioread16(&hw->common_cfg->queue_size);
return queue_size;
}
-/* This function returns the max allowed safe size for
- * all virtqueues. It is the minimal size that can be
- * suppprted by all virtqueues.
- */
u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw)
{
u16 queue_size, max_size, qid;
if (!queue_size)
continue;
- max_size = min(queue_size, max_size);
+ max_size = max(queue_size, max_size);
}
return max_size;
#define IFCVF_PCI_MAX_RESOURCE 6
#define IFCVF_LM_BAR 4
+#define IFCVF_MIN_VQ_SIZE 64
#define IFCVF_ERR(pdev, fmt, ...) dev_err(&pdev->dev, fmt, ##__VA_ARGS__)
#define IFCVF_DBG(pdev, fmt, ...) dev_dbg(&pdev->dev, fmt, ##__VA_ARGS__)
void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features);
u64 ifcvf_get_driver_features(struct ifcvf_hw *hw);
u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw);
+u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid);
#endif /* _IFCVF_H_ */
return ifcvf_get_max_vq_size(vf);
}
+static u16 ifcvf_vdpa_get_vq_num_min(struct vdpa_device *vdpa_dev)
+{
+ return IFCVF_MIN_VQ_SIZE;
+}
+
static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
struct vdpa_vq_state *state)
{
return -EINVAL;
}
+static u16 ifcvf_vdpa_get_vq_size(struct vdpa_device *vdpa_dev,
+ u16 qid)
+{
+ struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+
+ return ifcvf_get_vq_size(vf, qid);
+}
+
static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
u16 idx)
{
.set_status = ifcvf_vdpa_set_status,
.reset = ifcvf_vdpa_reset,
.get_vq_num_max = ifcvf_vdpa_get_vq_num_max,
+ .get_vq_num_min = ifcvf_vdpa_get_vq_num_min,
.get_vq_state = ifcvf_vdpa_get_vq_state,
.set_vq_state = ifcvf_vdpa_set_vq_state,
.set_vq_cb = ifcvf_vdpa_set_vq_cb,
.set_vq_num = ifcvf_vdpa_set_vq_num,
.set_vq_address = ifcvf_vdpa_set_vq_address,
.get_vq_irq = ifcvf_vdpa_get_vq_irq,
+ .get_vq_size = ifcvf_vdpa_get_vq_size,
.kick_vq = ifcvf_vdpa_kick_vq,
.get_generation = ifcvf_vdpa_get_generation,
.get_device_id = ifcvf_vdpa_get_device_id,
static bool mlx5_vdpa_debug;
-#define MLX5_CVQ_MAX_ENT 16
-
#define MLX5_LOG_VIO_FLAG(_feature) \
do { \
if (features & BIT_ULL(_feature)) \
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
struct mlx5_vdpa_virtqueue *mvq;
- if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx))
+ if (!is_index_valid(mvdev, idx))
return;
+ if (is_ctrl_vq_idx(mvdev, idx)) {
+ struct mlx5_control_vq *cvq = &mvdev->cvq;
+
+ cvq->vring.vring.num = num;
+ return;
+ }
+
mvq = &ndev->vqs[idx];
mvq->num_ent = num;
}
u16 idx = cvq->vring.last_avail_idx;
err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features,
- MLX5_CVQ_MAX_ENT, false,
+ cvq->vring.vring.num, false,
(struct vring_desc *)(uintptr_t)cvq->desc_addr,
(struct vring_avail *)(uintptr_t)cvq->driver_addr,
(struct vring_used *)(uintptr_t)cvq->device_addr);
struct device *dev = &aux_dev->dev;
vdpa_mgmtdev_unregister(&vdpa_aux->vdpa_mdev);
+ pds_vdpa_release_irqs(vdpa_aux->pdsv);
vp_modern_remove(&vdpa_aux->vd_mdev);
- pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
pds_vdpa_debugfs_del_vdpadev(vdpa_aux);
kfree(vdpa_aux);
return err;
}
-static void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv)
+void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv)
{
- struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
- struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux;
+ struct pds_vdpa_aux *vdpa_aux;
+ struct pci_dev *pdev;
int qid;
+ if (!pdsv)
+ return;
+
+ pdev = pdsv->vdpa_aux->padev->vf_pdev;
+ vdpa_aux = pdsv->vdpa_aux;
+
if (!vdpa_aux->nintrs)
return;
struct device *dma_dev;
struct pci_dev *pdev;
struct device *dev;
+ u8 status;
int err;
int i;
dma_dev = &pdev->dev;
pdsv->vdpa_dev.dma_dev = dma_dev;
+ status = pds_vdpa_get_status(&pdsv->vdpa_dev);
+ if (status == 0xff) {
+ dev_err(dev, "Broken PCI - status %#x\n", status);
+ err = -ENXIO;
+ goto err_unmap;
+ }
+
pdsv->supported_features = mgmt->supported_features;
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
#define PDS_VDPA_PACKED_INVERT_IDX 0x8000
+void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv);
int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux);
#endif /* _VDPA_DEV_H_ */
};
__ATTRIBUTE_GROUPS(vdpa_dev);
-static struct bus_type vdpa_bus = {
+static const struct bus_type vdpa_bus = {
.name = "vdpa",
.dev_groups = vdpa_dev_groups,
.match = vdpa_dev_match,
return vdpa_dev_net_mq_config_fill(msg, features_device, &config);
}
+static int
+vdpa_dev_blk_capacity_config_fill(struct sk_buff *msg,
+ const struct virtio_blk_config *config)
+{
+ u64 val_u64;
+
+ val_u64 = __virtio64_to_cpu(true, config->capacity);
+
+ return nla_put_u64_64bit(msg, VDPA_ATTR_DEV_BLK_CFG_CAPACITY,
+ val_u64, VDPA_ATTR_PAD);
+}
+
+static int
+vdpa_dev_blk_seg_size_config_fill(struct sk_buff *msg, u64 features,
+ const struct virtio_blk_config *config)
+{
+ u32 val_u32;
+
+ if ((features & BIT_ULL(VIRTIO_BLK_F_SIZE_MAX)) == 0)
+ return 0;
+
+ val_u32 = __virtio32_to_cpu(true, config->size_max);
+
+ return nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_SEG_SIZE, val_u32);
+}
+
+/* fill the block size*/
+static int
+vdpa_dev_blk_block_size_config_fill(struct sk_buff *msg, u64 features,
+ const struct virtio_blk_config *config)
+{
+ u32 val_u32;
+
+ if ((features & BIT_ULL(VIRTIO_BLK_F_BLK_SIZE)) == 0)
+ return 0;
+
+ val_u32 = __virtio32_to_cpu(true, config->blk_size);
+
+ return nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_BLK_SIZE, val_u32);
+}
+
+static int
+vdpa_dev_blk_seg_max_config_fill(struct sk_buff *msg, u64 features,
+ const struct virtio_blk_config *config)
+{
+ u32 val_u32;
+
+ if ((features & BIT_ULL(VIRTIO_BLK_F_SEG_MAX)) == 0)
+ return 0;
+
+ val_u32 = __virtio32_to_cpu(true, config->seg_max);
+
+ return nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_SEG_MAX, val_u32);
+}
+
+static int vdpa_dev_blk_mq_config_fill(struct sk_buff *msg, u64 features,
+ const struct virtio_blk_config *config)
+{
+ u16 val_u16;
+
+ if ((features & BIT_ULL(VIRTIO_BLK_F_MQ)) == 0)
+ return 0;
+
+ val_u16 = __virtio16_to_cpu(true, config->num_queues);
+
+ return nla_put_u16(msg, VDPA_ATTR_DEV_BLK_CFG_NUM_QUEUES, val_u16);
+}
+
+static int vdpa_dev_blk_topology_config_fill(struct sk_buff *msg, u64 features,
+ const struct virtio_blk_config *config)
+{
+ u16 min_io_size;
+ u32 opt_io_size;
+
+ if ((features & BIT_ULL(VIRTIO_BLK_F_TOPOLOGY)) == 0)
+ return 0;
+
+ min_io_size = __virtio16_to_cpu(true, config->min_io_size);
+ opt_io_size = __virtio32_to_cpu(true, config->opt_io_size);
+
+ if (nla_put_u8(msg, VDPA_ATTR_DEV_BLK_CFG_PHY_BLK_EXP,
+ config->physical_block_exp))
+ return -EMSGSIZE;
+
+ if (nla_put_u8(msg, VDPA_ATTR_DEV_BLK_CFG_ALIGN_OFFSET,
+ config->alignment_offset))
+ return -EMSGSIZE;
+
+ if (nla_put_u16(msg, VDPA_ATTR_DEV_BLK_CFG_MIN_IO_SIZE, min_io_size))
+ return -EMSGSIZE;
+
+ if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_OPT_IO_SIZE, opt_io_size))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int vdpa_dev_blk_discard_config_fill(struct sk_buff *msg, u64 features,
+ const struct virtio_blk_config *config)
+{
+ u32 val_u32;
+
+ if ((features & BIT_ULL(VIRTIO_BLK_F_DISCARD)) == 0)
+ return 0;
+
+ val_u32 = __virtio32_to_cpu(true, config->max_discard_sectors);
+ if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEC, val_u32))
+ return -EMSGSIZE;
+
+ val_u32 = __virtio32_to_cpu(true, config->max_discard_seg);
+ if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEG, val_u32))
+ return -EMSGSIZE;
+
+ val_u32 = __virtio32_to_cpu(true, config->discard_sector_alignment);
+ if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_DISCARD_SEC_ALIGN, val_u32))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int
+vdpa_dev_blk_write_zeroes_config_fill(struct sk_buff *msg, u64 features,
+ const struct virtio_blk_config *config)
+{
+ u32 val_u32;
+
+ if ((features & BIT_ULL(VIRTIO_BLK_F_WRITE_ZEROES)) == 0)
+ return 0;
+
+ val_u32 = __virtio32_to_cpu(true, config->max_write_zeroes_sectors);
+ if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEC, val_u32))
+ return -EMSGSIZE;
+
+ val_u32 = __virtio32_to_cpu(true, config->max_write_zeroes_seg);
+ if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEG, val_u32))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int vdpa_dev_blk_ro_config_fill(struct sk_buff *msg, u64 features)
+{
+ u8 ro;
+
+ ro = ((features & BIT_ULL(VIRTIO_BLK_F_RO)) == 0) ? 0 : 1;
+ if (nla_put_u8(msg, VDPA_ATTR_DEV_BLK_CFG_READ_ONLY, ro))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int vdpa_dev_blk_flush_config_fill(struct sk_buff *msg, u64 features)
+{
+ u8 flush;
+
+ flush = ((features & BIT_ULL(VIRTIO_BLK_F_FLUSH)) == 0) ? 0 : 1;
+ if (nla_put_u8(msg, VDPA_ATTR_DEV_BLK_CFG_FLUSH, flush))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev,
+ struct sk_buff *msg)
+{
+ struct virtio_blk_config config = {};
+ u64 features_device;
+
+ vdev->config->get_config(vdev, 0, &config, sizeof(config));
+
+ features_device = vdev->config->get_device_features(vdev);
+
+ if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_FEATURES, features_device,
+ VDPA_ATTR_PAD))
+ return -EMSGSIZE;
+
+ if (vdpa_dev_blk_capacity_config_fill(msg, &config))
+ return -EMSGSIZE;
+
+ if (vdpa_dev_blk_seg_size_config_fill(msg, features_device, &config))
+ return -EMSGSIZE;
+
+ if (vdpa_dev_blk_block_size_config_fill(msg, features_device, &config))
+ return -EMSGSIZE;
+
+ if (vdpa_dev_blk_seg_max_config_fill(msg, features_device, &config))
+ return -EMSGSIZE;
+
+ if (vdpa_dev_blk_mq_config_fill(msg, features_device, &config))
+ return -EMSGSIZE;
+
+ if (vdpa_dev_blk_topology_config_fill(msg, features_device, &config))
+ return -EMSGSIZE;
+
+ if (vdpa_dev_blk_discard_config_fill(msg, features_device, &config))
+ return -EMSGSIZE;
+
+ if (vdpa_dev_blk_write_zeroes_config_fill(msg, features_device, &config))
+ return -EMSGSIZE;
+
+ if (vdpa_dev_blk_ro_config_fill(msg, features_device))
+ return -EMSGSIZE;
+
+ if (vdpa_dev_blk_flush_config_fill(msg, features_device))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
static int
vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq,
int flags, struct netlink_ext_ack *extack)
case VIRTIO_ID_NET:
err = vdpa_dev_net_config_fill(vdev, msg);
break;
+ case VIRTIO_ID_BLOCK:
+ err = vdpa_dev_blk_config_fill(vdev, msg);
+ break;
default:
err = -EOPNOTSUPP;
break;
}
}
- vdpasim->running = true;
+ vdpasim->running = false;
spin_unlock(&vdpasim->iommu_lock);
vdpasim->features = 0;
vq->num = num;
}
+static u16 vdpasim_get_vq_size(struct vdpa_device *vdpa, u16 idx)
+{
+ struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+ struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+
+ if (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)
+ return vq->num;
+ else
+ return VDPASIM_QUEUE_MAX;
+}
+
static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
{
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
mutex_lock(&vdpasim->mutex);
vdpasim->status = status;
+ vdpasim->running = (status & VIRTIO_CONFIG_S_DRIVER_OK) != 0;
mutex_unlock(&vdpasim->mutex);
}
.get_driver_features = vdpasim_get_driver_features,
.set_config_cb = vdpasim_set_config_cb,
.get_vq_num_max = vdpasim_get_vq_num_max,
+ .get_vq_size = vdpasim_get_vq_size,
.get_device_id = vdpasim_get_device_id,
.get_vendor_id = vdpasim_get_vendor_id,
.get_status = vdpasim_get_status,
free_iova_fast(iovad, iova >> shift, iova_len);
}
+void vduse_domain_sync_single_for_device(struct vduse_iova_domain *domain,
+ dma_addr_t dma_addr, size_t size,
+ enum dma_data_direction dir)
+{
+ read_lock(&domain->bounce_lock);
+ if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
+ vduse_domain_bounce(domain, dma_addr, size, DMA_TO_DEVICE);
+ read_unlock(&domain->bounce_lock);
+}
+
+void vduse_domain_sync_single_for_cpu(struct vduse_iova_domain *domain,
+ dma_addr_t dma_addr, size_t size,
+ enum dma_data_direction dir)
+{
+ read_lock(&domain->bounce_lock);
+ if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
+ vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
+ read_unlock(&domain->bounce_lock);
+}
+
dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
struct page *page, unsigned long offset,
size_t size, enum dma_data_direction dir,
if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa))
goto err_unlock;
- if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
+ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
+ (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE);
read_unlock(&domain->bounce_lock);
enum dma_data_direction dir, unsigned long attrs)
{
struct iova_domain *iovad = &domain->stream_iovad;
-
read_lock(&domain->bounce_lock);
- if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
+ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
+ (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);
void vduse_domain_clear_map(struct vduse_iova_domain *domain,
struct vhost_iotlb *iotlb);
+void vduse_domain_sync_single_for_device(struct vduse_iova_domain *domain,
+ dma_addr_t dma_addr, size_t size,
+ enum dma_data_direction dir);
+
+void vduse_domain_sync_single_for_cpu(struct vduse_iova_domain *domain,
+ dma_addr_t dma_addr, size_t size,
+ enum dma_data_direction dir);
+
dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
struct page *page, unsigned long offset,
size_t size, enum dma_data_direction dir,
vq->num = num;
}
+static u16 vduse_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 idx)
+{
+ struct vduse_dev *dev = vdpa_to_vduse(vdpa);
+ struct vduse_virtqueue *vq = dev->vqs[idx];
+
+ if (vq->num)
+ return vq->num;
+ else
+ return vq->num_max;
+}
+
static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
u16 idx, bool ready)
{
.kick_vq = vduse_vdpa_kick_vq,
.set_vq_cb = vduse_vdpa_set_vq_cb,
.set_vq_num = vduse_vdpa_set_vq_num,
+ .get_vq_size = vduse_vdpa_get_vq_size,
.set_vq_ready = vduse_vdpa_set_vq_ready,
.get_vq_ready = vduse_vdpa_get_vq_ready,
.set_vq_state = vduse_vdpa_set_vq_state,
.free = vduse_vdpa_free,
};
+static void vduse_dev_sync_single_for_device(struct device *dev,
+ dma_addr_t dma_addr, size_t size,
+ enum dma_data_direction dir)
+{
+ struct vduse_dev *vdev = dev_to_vduse(dev);
+ struct vduse_iova_domain *domain = vdev->domain;
+
+ vduse_domain_sync_single_for_device(domain, dma_addr, size, dir);
+}
+
+static void vduse_dev_sync_single_for_cpu(struct device *dev,
+ dma_addr_t dma_addr, size_t size,
+ enum dma_data_direction dir)
+{
+ struct vduse_dev *vdev = dev_to_vduse(dev);
+ struct vduse_iova_domain *domain = vdev->domain;
+
+ vduse_domain_sync_single_for_cpu(domain, dma_addr, size, dir);
+}
+
static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size,
enum dma_data_direction dir,
}
static const struct dma_map_ops vduse_dev_dma_ops = {
+ .sync_single_for_device = vduse_dev_sync_single_for_device,
+ .sync_single_for_cpu = vduse_dev_sync_single_for_cpu,
.map_page = vduse_dev_map_page,
.unmap_page = vduse_dev_unmap_page,
.alloc = vduse_dev_alloc_coherent,
vp_modern_set_queue_size(mdev, qid, num);
}
+static u16 vp_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 qid)
+{
+ struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
+
+ return vp_modern_get_queue_size(mdev, qid);
+}
+
static int vp_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 qid,
u64 desc_area, u64 driver_area,
u64 device_area)
.set_vq_ready = vp_vdpa_set_vq_ready,
.get_vq_ready = vp_vdpa_get_vq_ready,
.set_vq_num = vp_vdpa_set_vq_num,
+ .get_vq_size = vp_vdpa_get_vq_size,
.set_vq_address = vp_vdpa_set_vq_address,
.kick_vq = vp_vdpa_kick_vq,
.get_generation = vp_vdpa_get_generation,
hdr = buf;
gso = &hdr->gso;
+ if (!sock_hlen)
+ memset(buf, 0, pad);
+
if ((gso->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
vhost16_to_cpu(vq, gso->csum_start) +
vhost16_to_cpu(vq, gso->csum_offset) + 2 >
const struct vdpa_config_ops *ops = vdpa->config;
int ret;
+ if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK))
+ return 0;
+
if (!ops->suspend)
return -EOPNOTSUPP;
const struct vdpa_config_ops *ops = vdpa->config;
int ret;
+ if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK))
+ return 0;
+
if (!ops->resume)
return -EOPNOTSUPP;
if (!ops->set_group_asid)
return -EOPNOTSUPP;
return ops->set_group_asid(vdpa, idx, s.num);
+ case VHOST_VDPA_GET_VRING_SIZE:
+ if (!ops->get_vq_size)
+ return -EOPNOTSUPP;
+ s.index = idx;
+ s.num = ops->get_vq_size(vdpa, idx);
+ if (copy_to_user(argp, &s, sizeof(s)))
+ return -EFAULT;
+ return 0;
case VHOST_GET_VRING_BASE:
r = ops->get_vq_state(v->vdpa, idx, &vq_state);
if (r)
of_node_put(dev->dev.of_node);
}
-static struct bus_type virtio_bus = {
+static const struct bus_type virtio_bus = {
.name = "virtio",
.match = virtio_dev_match,
.dev_groups = virtio_dev_groups,
if (drv && drv->freeze) {
ret = drv->freeze(dev);
- if (ret)
+ if (ret) {
+ virtio_config_enable(dev);
return ret;
+ }
}
if (dev->config->destroy_avq)
sizeof(struct vring_packed_desc));
vq->packed.vring.desc[head].id = cpu_to_le16(id);
- if (vq->do_unmap) {
+ if (vq->use_dma_api) {
vq->packed.desc_extra[id].addr = addr;
vq->packed.desc_extra[id].len = total_sg *
sizeof(struct vring_packed_desc);
desc[i].len = cpu_to_le32(sg->length);
desc[i].id = cpu_to_le16(id);
- if (unlikely(vq->do_unmap)) {
+ if (unlikely(vq->use_dma_api)) {
vq->packed.desc_extra[curr].addr = addr;
vq->packed.desc_extra[curr].len = sg->length;
vq->packed.desc_extra[curr].flags =
vq->free_head = id;
vq->vq.num_free += state->num;
- if (unlikely(vq->do_unmap)) {
+ if (unlikely(vq->use_dma_api)) {
curr = id;
for (i = 0; i < state->num; i++) {
vring_unmap_extra_packed(vq,
info = kmalloc(sizeof(*info), GFP_KERNEL);
if (!info)
return ERR_PTR(-ENOMEM);
+ if (ops->get_vq_size)
+ max_num = ops->get_vq_size(vdpa, index);
+ else
+ max_num = ops->get_vq_num_max(vdpa);
- max_num = ops->get_vq_num_max(vdpa);
if (max_num == 0) {
err = -ENOENT;
goto error_new_virtqueue;
#include <linux/interrupt.h>
#include <linux/vhost_iotlb.h>
#include <linux/virtio_net.h>
+#include <linux/virtio_blk.h>
#include <linux/if_ether.h>
/**
* @idx: virtqueue index
* Returns int: irq number of a virtqueue,
* negative number if no irq assigned.
+ * @get_vq_size: Get the size of a specific virtqueue (optional)
+ * @vdev: vdpa device
+ * @idx: virtqueue index
+ * Return u16: the size of the virtqueue
* @get_vq_align: Get the virtqueue align requirement
* for the device
* @vdev: vdpa device
(*get_vq_notification)(struct vdpa_device *vdev, u16 idx);
/* vq irq is not expected to be changed once DRIVER_OK is set */
int (*get_vq_irq)(struct vdpa_device *vdev, u16 idx);
+ u16 (*get_vq_size)(struct vdpa_device *vdev, u16 idx);
/* Device ops */
u32 (*get_vq_align)(struct vdpa_device *vdev);
/* virtio features that are provisioned to the vDPA device */
VDPA_ATTR_DEV_FEATURES, /* u64 */
+ VDPA_ATTR_DEV_BLK_CFG_CAPACITY, /* u64 */
+ VDPA_ATTR_DEV_BLK_CFG_SEG_SIZE, /* u32 */
+ VDPA_ATTR_DEV_BLK_CFG_BLK_SIZE, /* u32 */
+ VDPA_ATTR_DEV_BLK_CFG_SEG_MAX, /* u32 */
+ VDPA_ATTR_DEV_BLK_CFG_NUM_QUEUES, /* u16 */
+ VDPA_ATTR_DEV_BLK_CFG_PHY_BLK_EXP, /* u8 */
+ VDPA_ATTR_DEV_BLK_CFG_ALIGN_OFFSET, /* u8 */
+ VDPA_ATTR_DEV_BLK_CFG_MIN_IO_SIZE, /* u16 */
+ VDPA_ATTR_DEV_BLK_CFG_OPT_IO_SIZE, /* u32 */
+ VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEC, /* u32 */
+ VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEG, /* u32 */
+ VDPA_ATTR_DEV_BLK_CFG_DISCARD_SEC_ALIGN,/* u32 */
+ VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEC, /* u32 */
+ VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEG, /* u32 */
+ VDPA_ATTR_DEV_BLK_CFG_READ_ONLY, /* u8 */
+ VDPA_ATTR_DEV_BLK_CFG_FLUSH, /* u8 */
+
/* new attributes must be added above here */
VDPA_ATTR_MAX,
};
*/
#define VHOST_VDPA_GET_VRING_DESC_GROUP _IOWR(VHOST_VIRTIO, 0x7F, \
struct vhost_vring_state)
+
+/* Get the queue size of a specific virtqueue.
+ * userspace set the vring index in vhost_vring_state.index
+ * kernel set the queue size in vhost_vring_state.num
+ */
+#define VHOST_VDPA_GET_VRING_SIZE _IOWR(VHOST_VIRTIO, 0x80, \
+ struct vhost_vring_state)
#endif
#define VIRTIO_ADMIN_CMD_LEGACY_DEV_CFG_READ 0x5
#define VIRTIO_ADMIN_CMD_LEGACY_NOTIFY_INFO 0x6
-struct __packed virtio_admin_cmd_hdr {
+struct virtio_admin_cmd_hdr {
__le16 opcode;
/*
* 1 - SR-IOV
__le64 group_member_id;
};
-struct __packed virtio_admin_cmd_status {
+struct virtio_admin_cmd_status {
__le16 status;
__le16 status_qualifier;
/* Unused, reserved for future extensions. */
__u8 reserved2[4];
};
-struct __packed virtio_admin_cmd_legacy_wr_data {
+struct virtio_admin_cmd_legacy_wr_data {
__u8 offset; /* Starting offset of the register(s) to write. */
__u8 reserved[7];
__u8 registers[];
};
-struct __packed virtio_admin_cmd_legacy_rd_data {
+struct virtio_admin_cmd_legacy_rd_data {
__u8 offset; /* Starting offset of the register(s) to read. */
};
#define VIRTIO_ADMIN_CMD_MAX_NOTIFY_INFO 4
-struct __packed virtio_admin_cmd_notify_info_data {
+struct virtio_admin_cmd_notify_info_data {
__u8 flags; /* 0 = end of list, 1 = owner device, 2 = member device */
__u8 bar; /* BAR of the member or the owner device */
__u8 padding[6];
projects / linux-2.6-block.git / commitdiff