[linux-block.git] / drivers / net / ifb.c

// SPDX-License-Identifier: GPL-2.0-or-later
/* drivers/net/ifb.c:

	The purpose of this driver is to provide a device that allows
	for sharing of resources:

	1) qdiscs/policies that are per device as opposed to system wide.
	ifb allows for a device which can be redirected to thus providing
	an impression of sharing.

	2) Allows for queueing incoming traffic for shaping instead of
	dropping.

	The original concept is based on what is known as the IMQ
	driver initially written by Martin Devera, later rewritten
	by Patrick McHardy and then maintained by Andre Correa.

	You need the tc action  mirror or redirect to feed this device
	packets.


	Authors:	Jamal Hadi Salim (2005)

*/


#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/ethtool.h>
#include <linux/etherdevice.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/moduleparam.h>
#include <linux/netfilter_netdev.h>
#include <net/pkt_sched.h>
#include <net/net_namespace.h>

#define TX_Q_LIMIT    32

struct ifb_q_stats {
	u64 packets;
	u64 bytes;
	struct u64_stats_sync	sync;
};

struct ifb_q_private {
	struct net_device	*dev;
	struct tasklet_struct   ifb_tasklet;
	int			tasklet_pending;
	int			txqnum;
	struct sk_buff_head     rq;
	struct sk_buff_head     tq;
	struct ifb_q_stats	rx_stats;
	struct ifb_q_stats	tx_stats;
} ____cacheline_aligned_in_smp;

struct ifb_dev_private {
	struct ifb_q_private *tx_private;
};

/* For ethtools stats. */
struct ifb_q_stats_desc {
	char	desc[ETH_GSTRING_LEN];
	size_t	offset;
};

#define IFB_Q_STAT(m)	offsetof(struct ifb_q_stats, m)

static const struct ifb_q_stats_desc ifb_q_stats_desc[] = {
	{ "packets",	IFB_Q_STAT(packets) },
	{ "bytes",	IFB_Q_STAT(bytes) },
};

#define IFB_Q_STATS_LEN	ARRAY_SIZE(ifb_q_stats_desc)

static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev);
static int ifb_open(struct net_device *dev);
static int ifb_close(struct net_device *dev);

static void ifb_update_q_stats(struct ifb_q_stats *stats, int len)
{
	u64_stats_update_begin(&stats->sync);
	stats->packets++;
	stats->bytes += len;
	u64_stats_update_end(&stats->sync);
}

static void ifb_ri_tasklet(struct tasklet_struct *t)
{
	struct ifb_q_private *txp = from_tasklet(txp, t, ifb_tasklet);
	struct netdev_queue *txq;
	struct sk_buff *skb;

	txq = netdev_get_tx_queue(txp->dev, txp->txqnum);
	skb = skb_peek(&txp->tq);
	if (!skb) {
		if (!__netif_tx_trylock(txq))
			goto resched;
		skb_queue_splice_tail_init(&txp->rq, &txp->tq);
		__netif_tx_unlock(txq);
	}

	while ((skb = __skb_dequeue(&txp->tq)) != NULL) {
		/* Skip tc and netfilter to prevent redirection loop. */
		skb->redirected = 0;
#ifdef CONFIG_NET_CLS_ACT
		skb->tc_skip_classify = 1;
#endif
		nf_skip_egress(skb, true);

		ifb_update_q_stats(&txp->tx_stats, skb->len);

		rcu_read_lock();
		skb->dev = dev_get_by_index_rcu(dev_net(txp->dev), skb->skb_iif);
		if (!skb->dev) {
			rcu_read_unlock();
			dev_kfree_skb(skb);
			txp->dev->stats.tx_dropped++;
			if (skb_queue_len(&txp->tq) != 0)
				goto resched;
			break;
		}
		rcu_read_unlock();
		skb->skb_iif = txp->dev->ifindex;

		if (!skb->from_ingress) {
			dev_queue_xmit(skb);
		} else {
			skb_pull_rcsum(skb, skb->mac_len);
			netif_receive_skb(skb);
		}
	}

	if (__netif_tx_trylock(txq)) {
		skb = skb_peek(&txp->rq);
		if (!skb) {
			txp->tasklet_pending = 0;
			if (netif_tx_queue_stopped(txq))
				netif_tx_wake_queue(txq);
		} else {
			__netif_tx_unlock(txq);
			goto resched;
		}
		__netif_tx_unlock(txq);
	} else {
resched:
		txp->tasklet_pending = 1;
		tasklet_schedule(&txp->ifb_tasklet);
	}

}

static void ifb_stats64(struct net_device *dev,
			struct rtnl_link_stats64 *stats)
{
	struct ifb_dev_private *dp = netdev_priv(dev);
	struct ifb_q_private *txp = dp->tx_private;
	unsigned int start;
	u64 packets, bytes;
	int i;

	for (i = 0; i < dev->num_tx_queues; i++,txp++) {
		do {
			start = u64_stats_fetch_begin(&txp->rx_stats.sync);
			packets = txp->rx_stats.packets;
			bytes = txp->rx_stats.bytes;
		} while (u64_stats_fetch_retry(&txp->rx_stats.sync, start));
		stats->rx_packets += packets;
		stats->rx_bytes += bytes;

		do {
			start = u64_stats_fetch_begin(&txp->tx_stats.sync);
			packets = txp->tx_stats.packets;
			bytes = txp->tx_stats.bytes;
		} while (u64_stats_fetch_retry(&txp->tx_stats.sync, start));
		stats->tx_packets += packets;
		stats->tx_bytes += bytes;
	}
	stats->rx_dropped = dev->stats.rx_dropped;
	stats->tx_dropped = dev->stats.tx_dropped;
}

static int ifb_dev_init(struct net_device *dev)
{
	struct ifb_dev_private *dp = netdev_priv(dev);
	struct ifb_q_private *txp;
	int i;

	txp = kcalloc(dev->num_tx_queues, sizeof(*txp), GFP_KERNEL);
	if (!txp)
		return -ENOMEM;
	dp->tx_private = txp;
	for (i = 0; i < dev->num_tx_queues; i++,txp++) {
		txp->txqnum = i;
		txp->dev = dev;
		__skb_queue_head_init(&txp->rq);
		__skb_queue_head_init(&txp->tq);
		u64_stats_init(&txp->rx_stats.sync);
		u64_stats_init(&txp->tx_stats.sync);
		tasklet_setup(&txp->ifb_tasklet, ifb_ri_tasklet);
		netif_tx_start_queue(netdev_get_tx_queue(dev, i));
	}
	return 0;
}

static void ifb_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
{
	u8 *p = buf;
	int i, j;

	switch (stringset) {
	case ETH_SS_STATS:
		for (i = 0; i < dev->real_num_rx_queues; i++)
			for (j = 0; j < IFB_Q_STATS_LEN; j++)
				ethtool_sprintf(&p, "rx_queue_%u_%.18s",
						i, ifb_q_stats_desc[j].desc);

		for (i = 0; i < dev->real_num_tx_queues; i++)
			for (j = 0; j < IFB_Q_STATS_LEN; j++)
				ethtool_sprintf(&p, "tx_queue_%u_%.18s",
						i, ifb_q_stats_desc[j].desc);

		break;
	}
}

static int ifb_get_sset_count(struct net_device *dev, int sset)
{
	switch (sset) {
	case ETH_SS_STATS:
		return IFB_Q_STATS_LEN * (dev->real_num_rx_queues +
					  dev->real_num_tx_queues);
	default:
		return -EOPNOTSUPP;
	}
}

static void ifb_fill_stats_data(u64 **data,
				struct ifb_q_stats *q_stats)
{
	void *stats_base = (void *)q_stats;
	unsigned int start;
	size_t offset;
	int j;

	do {
		start = u64_stats_fetch_begin(&q_stats->sync);
		for (j = 0; j < IFB_Q_STATS_LEN; j++) {
			offset = ifb_q_stats_desc[j].offset;
			(*data)[j] = *(u64 *)(stats_base + offset);
		}
	} while (u64_stats_fetch_retry(&q_stats->sync, start));

	*data += IFB_Q_STATS_LEN;
}

static void ifb_get_ethtool_stats(struct net_device *dev,
				  struct ethtool_stats *stats, u64 *data)
{
	struct ifb_dev_private *dp = netdev_priv(dev);
	struct ifb_q_private *txp;
	int i;

	for (i = 0; i < dev->real_num_rx_queues; i++) {
		txp = dp->tx_private + i;
		ifb_fill_stats_data(&data, &txp->rx_stats);
	}

	for (i = 0; i < dev->real_num_tx_queues; i++) {
		txp = dp->tx_private + i;
		ifb_fill_stats_data(&data, &txp->tx_stats);
	}
}

static const struct net_device_ops ifb_netdev_ops = {
	.ndo_open	= ifb_open,
	.ndo_stop	= ifb_close,
	.ndo_get_stats64 = ifb_stats64,
	.ndo_start_xmit	= ifb_xmit,
	.ndo_validate_addr = eth_validate_addr,
	.ndo_init	= ifb_dev_init,
};

static const struct ethtool_ops ifb_ethtool_ops = {
	.get_strings		= ifb_get_strings,
	.get_sset_count		= ifb_get_sset_count,
	.get_ethtool_stats	= ifb_get_ethtool_stats,
};

#define IFB_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG  | NETIF_F_FRAGLIST	| \
		      NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL	| \
		      NETIF_F_HIGHDMA | NETIF_F_HW_VLAN_CTAG_TX		| \
		      NETIF_F_HW_VLAN_STAG_TX)

static void ifb_dev_free(struct net_device *dev)
{
	struct ifb_dev_private *dp = netdev_priv(dev);
	struct ifb_q_private *txp = dp->tx_private;
	int i;

	for (i = 0; i < dev->num_tx_queues; i++,txp++) {
		tasklet_kill(&txp->ifb_tasklet);
		__skb_queue_purge(&txp->rq);
		__skb_queue_purge(&txp->tq);
	}
	kfree(dp->tx_private);
}

static void ifb_setup(struct net_device *dev)
{
	/* Initialize the device structure. */
	dev->netdev_ops = &ifb_netdev_ops;
	dev->ethtool_ops = &ifb_ethtool_ops;

	/* Fill in device structure with ethernet-generic values. */
	ether_setup(dev);
	dev->tx_queue_len = TX_Q_LIMIT;

	dev->features |= IFB_FEATURES;
	dev->hw_features |= dev->features;
	dev->hw_enc_features |= dev->features;
	dev->vlan_features |= IFB_FEATURES & ~(NETIF_F_HW_VLAN_CTAG_TX |
					       NETIF_F_HW_VLAN_STAG_TX);

	dev->flags |= IFF_NOARP;
	dev->flags &= ~IFF_MULTICAST;
	dev->priv_flags &= ~IFF_TX_SKB_SHARING;
	netif_keep_dst(dev);
	eth_hw_addr_random(dev);
	dev->needs_free_netdev = true;
	dev->priv_destructor = ifb_dev_free;

	dev->min_mtu = 0;
	dev->max_mtu = 0;
}

static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct ifb_dev_private *dp = netdev_priv(dev);
	struct ifb_q_private *txp = dp->tx_private + skb_get_queue_mapping(skb);

	ifb_update_q_stats(&txp->rx_stats, skb->len);

	if (!skb->redirected || !skb->skb_iif) {
		dev_kfree_skb(skb);
		dev->stats.rx_dropped++;
		return NETDEV_TX_OK;
	}

	if (skb_queue_len(&txp->rq) >= dev->tx_queue_len)
		netif_tx_stop_queue(netdev_get_tx_queue(dev, txp->txqnum));

	__skb_queue_tail(&txp->rq, skb);
	if (!txp->tasklet_pending) {
		txp->tasklet_pending = 1;
		tasklet_schedule(&txp->ifb_tasklet);
	}

	return NETDEV_TX_OK;
}

static int ifb_close(struct net_device *dev)
{
	netif_tx_stop_all_queues(dev);
	return 0;
}

static int ifb_open(struct net_device *dev)
{
	netif_tx_start_all_queues(dev);
	return 0;
}

static int ifb_validate(struct nlattr *tb[], struct nlattr *data[],
			struct netlink_ext_ack *extack)
{
	if (tb[IFLA_ADDRESS]) {
		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)
			return -EINVAL;
		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))
			return -EADDRNOTAVAIL;
	}
	return 0;
}

static struct rtnl_link_ops ifb_link_ops __read_mostly = {
	.kind		= "ifb",
	.priv_size	= sizeof(struct ifb_dev_private),
	.setup		= ifb_setup,
	.validate	= ifb_validate,
};

/* Number of ifb devices to be set up by this module.
 * Note that these legacy devices have one queue.
 * Prefer something like : ip link add ifb10 numtxqueues 8 type ifb
 */
static int numifbs = 2;
module_param(numifbs, int, 0);
MODULE_PARM_DESC(numifbs, "Number of ifb devices");

static int __init ifb_init_one(int index)
{
	struct net_device *dev_ifb;
	int err;

	dev_ifb = alloc_netdev(sizeof(struct ifb_dev_private), "ifb%d",
			       NET_NAME_UNKNOWN, ifb_setup);

	if (!dev_ifb)
		return -ENOMEM;

	dev_ifb->rtnl_link_ops = &ifb_link_ops;
	err = register_netdevice(dev_ifb);
	if (err < 0)
		goto err;

	return 0;

err:
	free_netdev(dev_ifb);
	return err;
}

static int __init ifb_init_module(void)
{
	int i, err;

	down_write(&pernet_ops_rwsem);
	rtnl_lock();
	err = __rtnl_link_register(&ifb_link_ops);
	if (err < 0)
		goto out;

	for (i = 0; i < numifbs && !err; i++) {
		err = ifb_init_one(i);
		cond_resched();
	}
	if (err)
		__rtnl_link_unregister(&ifb_link_ops);

out:
	rtnl_unlock();
	up_write(&pernet_ops_rwsem);

	return err;
}

static void __exit ifb_cleanup_module(void)
{
	rtnl_link_unregister(&ifb_link_ops);
}

module_init(ifb_init_module);
module_exit(ifb_cleanup_module);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Intermediate Functional Block (ifb) netdevice driver for sharing of resources and ingress packet queuing");
MODULE_AUTHOR("Jamal Hadi Salim");
MODULE_ALIAS_RTNL_LINK("ifb");
Commit	Line	Data
2874c5fd	1	// SPDX-License-Identifier: GPL-2.0-or-later
6aa20a22	2	/* drivers/net/ifb.c:
253af423 JHS	3
	4	The purpose of this driver is to provide a device that allows
	5	for sharing of resources:
	6
	7	1) qdiscs/policies that are per device as opposed to system wide.
	8	ifb allows for a device which can be redirected to thus providing
	9	an impression of sharing.
	10
	11	2) Allows for queueing incoming traffic for shaping instead of
6aa20a22 JG	12	dropping.
6aa20a22 JG	13
253af423 JHS	14	The original concept is based on what is known as the IMQ
	15	driver initially written by Martin Devera, later rewritten
	16	by Patrick McHardy and then maintained by Andre Correa.
	17
	18	You need the tc action mirror or redirect to feed this device
cf9207d7	19	packets.
253af423	20
6aa20a22	21
cf9207d7	22	Authors: Jamal Hadi Salim (2005)
6aa20a22	23
253af423 JHS	24	*/
	25
	26
253af423 JHS	27	#include <linux/module.h>
	28	#include <linux/kernel.h>
	29	#include <linux/netdevice.h>
a21ee5b2	30	#include <linux/ethtool.h>
253af423 JHS	31	#include <linux/etherdevice.h>
253af423 JHS	32	#include <linux/init.h>
a6b7a407	33	#include <linux/interrupt.h>
253af423	34	#include <linux/moduleparam.h>
42df6e1d	35	#include <linux/netfilter_netdev.h>
6aa20a22	36	#include <net/pkt_sched.h>
881d966b	37	#include <net/net_namespace.h>
253af423	38
253af423	39	#define TX_Q_LIMIT 32
a21ee5b2 TZ	40
	41	struct ifb_q_stats {
	42	u64 packets;
	43	u64 bytes;
	44	struct u64_stats_sync sync;
	45	};
	46
9e29e21a ED	47	struct ifb_q_private {
9e29e21a ED	48	struct net_device *dev;
253af423	49	struct tasklet_struct ifb_tasklet;
9e29e21a ED	50	int tasklet_pending;
9e29e21a ED	51	int txqnum;
253af423 JHS	52	struct sk_buff_head rq;
253af423 JHS	53	struct sk_buff_head tq;
a21ee5b2 TZ	54	struct ifb_q_stats rx_stats;
a21ee5b2 TZ	55	struct ifb_q_stats tx_stats;
9e29e21a	56	} ____cacheline_aligned_in_smp;
253af423	57
9e29e21a ED	58	struct ifb_dev_private {
	59	struct ifb_q_private *tx_private;
	60	};
253af423	61
a21ee5b2 TZ	62	/* For ethtools stats. */
	63	struct ifb_q_stats_desc {
	64	char desc[ETH_GSTRING_LEN];
	65	size_t offset;
	66	};
	67
	68	#define IFB_Q_STAT(m) offsetof(struct ifb_q_stats, m)
	69
	70	static const struct ifb_q_stats_desc ifb_q_stats_desc[] = {
	71	{ "packets", IFB_Q_STAT(packets) },
	72	{ "bytes", IFB_Q_STAT(bytes) },
	73	};
	74
	75	#define IFB_Q_STATS_LEN ARRAY_SIZE(ifb_q_stats_desc)
	76
424efe9c	77	static netdev_tx_t ifb_xmit(struct sk_buff skb, struct net_device dev);
253af423 JHS	78	static int ifb_open(struct net_device *dev);
	79	static int ifb_close(struct net_device *dev);
	80
a21ee5b2 TZ	81	static void ifb_update_q_stats(struct ifb_q_stats *stats, int len)
	82	{
	83	u64_stats_update_begin(&stats->sync);
	84	stats->packets++;
	85	stats->bytes += len;
	86	u64_stats_update_end(&stats->sync);
	87	}
	88
08267523	89	static void ifb_ri_tasklet(struct tasklet_struct *t)
253af423	90	{
08267523	91	struct ifb_q_private *txp = from_tasklet(txp, t, ifb_tasklet);
c3f26a26	92	struct netdev_queue *txq;
253af423 JHS	93	struct sk_buff *skb;
253af423 JHS	94
9e29e21a ED	95	txq = netdev_get_tx_queue(txp->dev, txp->txqnum);
	96	skb = skb_peek(&txp->tq);
	97	if (!skb) {
	98	if (!__netif_tx_trylock(txq))
253af423	99	goto resched;
9e29e21a ED	100	skb_queue_splice_tail_init(&txp->rq, &txp->tq);
9e29e21a ED	101	__netif_tx_unlock(txq);
253af423 JHS	102	}
253af423 JHS	103
9e29e21a	104	while ((skb = __skb_dequeue(&txp->tq)) != NULL) {
42df6e1d	105	/* Skip tc and netfilter to prevent redirection loop. */
2c64605b	106	skb->redirected = 0;
7444d706	107	#ifdef CONFIG_NET_CLS_ACT
e7246e12	108	skb->tc_skip_classify = 1;
7444d706	109	#endif
42df6e1d	110	nf_skip_egress(skb, true);
3b0c9cbb	111
a21ee5b2	112	ifb_update_q_stats(&txp->tx_stats, skb->len);
c01003c2	113
05e8689c	114	rcu_read_lock();
9e29e21a	115	skb->dev = dev_get_by_index_rcu(dev_net(txp->dev), skb->skb_iif);
c01003c2	116	if (!skb->dev) {
05e8689c	117	rcu_read_unlock();
c01003c2	118	dev_kfree_skb(skb);
9e29e21a ED	119	txp->dev->stats.tx_dropped++;
9e29e21a ED	120	if (skb_queue_len(&txp->tq) != 0)
75c1c825	121	goto resched;
c01003c2 PM	122	break;
c01003c2 PM	123	}
05e8689c	124	rcu_read_unlock();
9e29e21a	125	skb->skb_iif = txp->dev->ifindex;
c01003c2	126
2c64605b	127	if (!skb->from_ingress) {
253af423	128	dev_queue_xmit(skb);
bc31c905	129	} else {
b1d2e4e0	130	skb_pull_rcsum(skb, skb->mac_len);
1a75972c	131	netif_receive_skb(skb);
bc31c905	132	}
253af423 JHS	133	}
253af423 JHS	134
c3f26a26	135	if (__netif_tx_trylock(txq)) {
9e29e21a ED	136	skb = skb_peek(&txp->rq);
	137	if (!skb) {
	138	txp->tasklet_pending = 0;
	139	if (netif_tx_queue_stopped(txq))
	140	netif_tx_wake_queue(txq);
253af423	141	} else {
c3f26a26	142	__netif_tx_unlock(txq);
253af423 JHS	143	goto resched;
253af423 JHS	144	}
c3f26a26	145	__netif_tx_unlock(txq);
253af423 JHS	146	} else {
253af423 JHS	147	resched:
9e29e21a ED	148	txp->tasklet_pending = 1;
9e29e21a ED	149	tasklet_schedule(&txp->ifb_tasklet);
253af423 JHS	150	}
	151
	152	}
	153
bc1f4470	154	static void ifb_stats64(struct net_device *dev,
bc1f4470	155	struct rtnl_link_stats64 *stats)
3b0c9cbb	156	{
9e29e21a ED	157	struct ifb_dev_private *dp = netdev_priv(dev);
9e29e21a ED	158	struct ifb_q_private *txp = dp->tx_private;
3b0c9cbb	159	unsigned int start;
9e29e21a ED	160	u64 packets, bytes;
	161	int i;
	162
	163	for (i = 0; i < dev->num_tx_queues; i++,txp++) {
	164	do {
068c38ad	165	start = u64_stats_fetch_begin(&txp->rx_stats.sync);
a21ee5b2 TZ	166	packets = txp->rx_stats.packets;
a21ee5b2 TZ	167	bytes = txp->rx_stats.bytes;
068c38ad	168	} while (u64_stats_fetch_retry(&txp->rx_stats.sync, start));
9e29e21a ED	169	stats->rx_packets += packets;
	170	stats->rx_bytes += bytes;
	171
	172	do {
068c38ad	173	start = u64_stats_fetch_begin(&txp->tx_stats.sync);
a21ee5b2 TZ	174	packets = txp->tx_stats.packets;
a21ee5b2 TZ	175	bytes = txp->tx_stats.bytes;
068c38ad	176	} while (u64_stats_fetch_retry(&txp->tx_stats.sync, start));
9e29e21a ED	177	stats->tx_packets += packets;
	178	stats->tx_bytes += bytes;
	179	}
3b0c9cbb	180	stats->rx_dropped = dev->stats.rx_dropped;
3b0c9cbb	181	stats->tx_dropped = dev->stats.tx_dropped;
3b0c9cbb	182	}
3b0c9cbb	183
9e29e21a ED	184	static int ifb_dev_init(struct net_device *dev)
	185	{
	186	struct ifb_dev_private *dp = netdev_priv(dev);
	187	struct ifb_q_private *txp;
	188	int i;
	189
	190	txp = kcalloc(dev->num_tx_queues, sizeof(*txp), GFP_KERNEL);
	191	if (!txp)
	192	return -ENOMEM;
	193	dp->tx_private = txp;
	194	for (i = 0; i < dev->num_tx_queues; i++,txp++) {
	195	txp->txqnum = i;
	196	txp->dev = dev;
	197	__skb_queue_head_init(&txp->rq);
	198	__skb_queue_head_init(&txp->tq);
a21ee5b2 TZ	199	u64_stats_init(&txp->rx_stats.sync);
a21ee5b2 TZ	200	u64_stats_init(&txp->tx_stats.sync);
08267523	201	tasklet_setup(&txp->ifb_tasklet, ifb_ri_tasklet);
9e29e21a ED	202	netif_tx_start_queue(netdev_get_tx_queue(dev, i));
	203	}
	204	return 0;
	205	}
3b0c9cbb	206
a21ee5b2 TZ	207	static void ifb_get_strings(struct net_device dev, u32 stringset, u8 buf)
	208	{
	209	u8 *p = buf;
	210	int i, j;
	211
	212	switch (stringset) {
	213	case ETH_SS_STATS:
	214	for (i = 0; i < dev->real_num_rx_queues; i++)
	215	for (j = 0; j < IFB_Q_STATS_LEN; j++)
	216	ethtool_sprintf(&p, "rx_queue_%u_%.18s",
	217	i, ifb_q_stats_desc[j].desc);
	218
	219	for (i = 0; i < dev->real_num_tx_queues; i++)
	220	for (j = 0; j < IFB_Q_STATS_LEN; j++)
	221	ethtool_sprintf(&p, "tx_queue_%u_%.18s",
	222	i, ifb_q_stats_desc[j].desc);
	223
	224	break;
	225	}
	226	}
	227
	228	static int ifb_get_sset_count(struct net_device *dev, int sset)
	229	{
	230	switch (sset) {
	231	case ETH_SS_STATS:
	232	return IFB_Q_STATS_LEN * (dev->real_num_rx_queues +
	233	dev->real_num_tx_queues);
	234	default:
	235	return -EOPNOTSUPP;
	236	}
	237	}
	238
	239	static void ifb_fill_stats_data(u64 **data,
	240	struct ifb_q_stats *q_stats)
	241	{
	242	void stats_base = (void )q_stats;
	243	unsigned int start;
	244	size_t offset;
	245	int j;
	246
	247	do {
068c38ad	248	start = u64_stats_fetch_begin(&q_stats->sync);
a21ee5b2 TZ	249	for (j = 0; j < IFB_Q_STATS_LEN; j++) {
	250	offset = ifb_q_stats_desc[j].offset;
	251	(data)[j] = (u64 *)(stats_base + offset);
	252	}
068c38ad	253	} while (u64_stats_fetch_retry(&q_stats->sync, start));
a21ee5b2 TZ	254
	255	*data += IFB_Q_STATS_LEN;
	256	}
	257
	258	static void ifb_get_ethtool_stats(struct net_device *dev,
	259	struct ethtool_stats stats, u64 data)
	260	{
	261	struct ifb_dev_private *dp = netdev_priv(dev);
	262	struct ifb_q_private *txp;
	263	int i;
	264
	265	for (i = 0; i < dev->real_num_rx_queues; i++) {
	266	txp = dp->tx_private + i;
	267	ifb_fill_stats_data(&data, &txp->rx_stats);
	268	}
	269
	270	for (i = 0; i < dev->real_num_tx_queues; i++) {
	271	txp = dp->tx_private + i;
	272	ifb_fill_stats_data(&data, &txp->tx_stats);
	273	}
	274	}
	275
8dfcdf34	276	static const struct net_device_ops ifb_netdev_ops = {
8dfcdf34 SH	277	.ndo_open = ifb_open,
8dfcdf34 SH	278	.ndo_stop = ifb_close,
3b0c9cbb	279	.ndo_get_stats64 = ifb_stats64,
00829823 SH	280	.ndo_start_xmit = ifb_xmit,
00829823 SH	281	.ndo_validate_addr = eth_validate_addr,
9e29e21a	282	.ndo_init = ifb_dev_init,
8dfcdf34 SH	283	};
8dfcdf34 SH	284
a21ee5b2 TZ	285	static const struct ethtool_ops ifb_ethtool_ops = {
	286	.get_strings = ifb_get_strings,
	287	.get_sset_count = ifb_get_sset_count,
	288	.get_ethtool_stats = ifb_get_ethtool_stats,
	289	};
	290
34324dc2	291	#define IFB_FEATURES (NETIF_F_HW_CSUM \| NETIF_F_SG \| NETIF_F_FRAGLIST \| \
ecb8fed4	292	NETIF_F_GSO_SOFTWARE \| NETIF_F_GSO_ENCAP_ALL \| \
28d2b136 PM	293	NETIF_F_HIGHDMA \| NETIF_F_HW_VLAN_CTAG_TX \| \
28d2b136 PM	294	NETIF_F_HW_VLAN_STAG_TX)
39980292	295
9e29e21a ED	296	static void ifb_dev_free(struct net_device *dev)
	297	{
	298	struct ifb_dev_private *dp = netdev_priv(dev);
	299	struct ifb_q_private *txp = dp->tx_private;
	300	int i;
	301
	302	for (i = 0; i < dev->num_tx_queues; i++,txp++) {
	303	tasklet_kill(&txp->ifb_tasklet);
	304	__skb_queue_purge(&txp->rq);
	305	__skb_queue_purge(&txp->tq);
	306	}
	307	kfree(dp->tx_private);
9e29e21a ED	308	}
9e29e21a ED	309
9ba2cd65	310	static void ifb_setup(struct net_device *dev)
253af423 JHS	311	{
253af423 JHS	312	/* Initialize the device structure. */
8dfcdf34	313	dev->netdev_ops = &ifb_netdev_ops;
a21ee5b2	314	dev->ethtool_ops = &ifb_ethtool_ops;
253af423 JHS	315
	316	/* Fill in device structure with ethernet-generic values. */
	317	ether_setup(dev);
	318	dev->tx_queue_len = TX_Q_LIMIT;
8dfcdf34	319
39980292	320	dev->features \|= IFB_FEATURES;
7d945796 ED	321	dev->hw_features \|= dev->features;
7d945796 ED	322	dev->hw_enc_features \|= dev->features;
8dd6e147 VY	323	dev->vlan_features \|= IFB_FEATURES & ~(NETIF_F_HW_VLAN_CTAG_TX \|
8dd6e147 VY	324	NETIF_F_HW_VLAN_STAG_TX);
39980292	325
253af423 JHS	326	dev->flags \|= IFF_NOARP;
253af423 JHS	327	dev->flags &= ~IFF_MULTICAST;
02875878 ED	328	dev->priv_flags &= ~IFF_TX_SKB_SHARING;
02875878 ED	329	netif_keep_dst(dev);
f2cedb63	330	eth_hw_addr_random(dev);
cf124db5 DM	331	dev->needs_free_netdev = true;
cf124db5 DM	332	dev->priv_destructor = ifb_dev_free;
e94cd811 ZS	333
	334	dev->min_mtu = 0;
	335	dev->max_mtu = 0;
253af423 JHS	336	}
253af423 JHS	337
424efe9c	338	static netdev_tx_t ifb_xmit(struct sk_buff skb, struct net_device dev)
253af423	339	{
9e29e21a	340	struct ifb_dev_private *dp = netdev_priv(dev);
9e29e21a	341	struct ifb_q_private *txp = dp->tx_private + skb_get_queue_mapping(skb);
253af423	342
a21ee5b2	343	ifb_update_q_stats(&txp->rx_stats, skb->len);
253af423	344
2c64605b	345	if (!skb->redirected \|\| !skb->skb_iif) {
253af423	346	dev_kfree_skb(skb);
3b0c9cbb	347	dev->stats.rx_dropped++;
424efe9c	348	return NETDEV_TX_OK;
253af423 JHS	349	}
253af423 JHS	350
9e29e21a ED	351	if (skb_queue_len(&txp->rq) >= dev->tx_queue_len)
9e29e21a ED	352	netif_tx_stop_queue(netdev_get_tx_queue(dev, txp->txqnum));
253af423	353
9e29e21a ED	354	__skb_queue_tail(&txp->rq, skb);
	355	if (!txp->tasklet_pending) {
	356	txp->tasklet_pending = 1;
	357	tasklet_schedule(&txp->ifb_tasklet);
253af423 JHS	358	}
253af423 JHS	359
424efe9c	360	return NETDEV_TX_OK;
253af423 JHS	361	}
253af423 JHS	362
253af423 JHS	363	static int ifb_close(struct net_device *dev)
253af423 JHS	364	{
9e29e21a	365	netif_tx_stop_all_queues(dev);
253af423 JHS	366	return 0;
	367	}
	368
	369	static int ifb_open(struct net_device *dev)
	370	{
9e29e21a	371	netif_tx_start_all_queues(dev);
253af423 JHS	372	return 0;
	373	}
	374
a8b8a889 MS	375	static int ifb_validate(struct nlattr tb[], struct nlattr data[],
a8b8a889 MS	376	struct netlink_ext_ack *extack)
0e06877c PM	377	{
	378	if (tb[IFLA_ADDRESS]) {
	379	if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)
	380	return -EINVAL;
	381	if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))
	382	return -EADDRNOTAVAIL;
	383	}
	384	return 0;
	385	}
	386
9ba2cd65 PM	387	static struct rtnl_link_ops ifb_link_ops __read_mostly = {
9ba2cd65 PM	388	.kind = "ifb",
9e29e21a	389	.priv_size = sizeof(struct ifb_dev_private),
9ba2cd65	390	.setup = ifb_setup,
0e06877c	391	.validate = ifb_validate,
9ba2cd65 PM	392	};
9ba2cd65 PM	393
9e29e21a ED	394	/* Number of ifb devices to be set up by this module.
	395	* Note that these legacy devices have one queue.
	396	* Prefer something like : ip link add ifb10 numtxqueues 8 type ifb
	397	*/
	398	static int numifbs = 2;
2d85cba2 PM	399	module_param(numifbs, int, 0);
	400	MODULE_PARM_DESC(numifbs, "Number of ifb devices");
	401
253af423 JHS	402	static int __init ifb_init_one(int index)
	403	{
	404	struct net_device *dev_ifb;
	405	int err;
	406
9e29e21a	407	dev_ifb = alloc_netdev(sizeof(struct ifb_dev_private), "ifb%d",
c835a677	408	NET_NAME_UNKNOWN, ifb_setup);
253af423 JHS	409
	410	if (!dev_ifb)
	411	return -ENOMEM;
	412
9ba2cd65 PM	413	dev_ifb->rtnl_link_ops = &ifb_link_ops;
	414	err = register_netdevice(dev_ifb);
	415	if (err < 0)
	416	goto err;
94833dfb	417
9ba2cd65	418	return 0;
62b7ffca	419
9ba2cd65 PM	420	err:
	421	free_netdev(dev_ifb);
	422	return err;
6aa20a22	423	}
253af423 JHS	424
253af423 JHS	425	static int __init ifb_init_module(void)
6aa20a22	426	{
9ba2cd65 PM	427	int i, err;
9ba2cd65 PM	428
554873e5	429	down_write(&pernet_ops_rwsem);
9ba2cd65 PM	430	rtnl_lock();
9ba2cd65 PM	431	err = __rtnl_link_register(&ifb_link_ops);
f2966cd5	432	if (err < 0)
f2966cd5	433	goto out;
62b7ffca	434
440d57bc	435	for (i = 0; i < numifbs && !err; i++) {
6aa20a22	436	err = ifb_init_one(i);
440d57bc	437	cond_resched();
440d57bc	438	}
2d85cba2	439	if (err)
9ba2cd65	440	__rtnl_link_unregister(&ifb_link_ops);
f2966cd5	441
f2966cd5	442	out:
9ba2cd65	443	rtnl_unlock();
554873e5	444	up_write(&pernet_ops_rwsem);
253af423 JHS	445
253af423 JHS	446	return err;
6aa20a22	447	}
253af423 JHS	448
	449	static void __exit ifb_cleanup_module(void)
	450	{
2d85cba2	451	rtnl_link_unregister(&ifb_link_ops);
253af423 JHS	452	}
	453
	454	module_init(ifb_init_module);
	455	module_exit(ifb_cleanup_module);
	456	MODULE_LICENSE("GPL");
55c90047	457	MODULE_DESCRIPTION("Intermediate Functional Block (ifb) netdevice driver for sharing of resources and ingress packet queuing");
253af423	458	MODULE_AUTHOR("Jamal Hadi Salim");
9ba2cd65	459	MODULE_ALIAS_RTNL_LINK("ifb");