[linux-2.6-block.git] / net / sched / sch_prio.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/sch_prio.c	Simple 3-band priority "scheduler".
 *
 * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 * Fixes:       19990609: J Hadi Salim <hadi@nortelnetworks.com>:
 *              Init --  EINVAL when opt undefined
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>

struct prio_sched_data {
	int bands;
	struct tcf_proto __rcu *filter_list;
	struct tcf_block *block;
	u8  prio2band[TC_PRIO_MAX+1];
	struct Qdisc *queues[TCQ_PRIO_BANDS];
};


static struct Qdisc *
prio_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
{
	struct prio_sched_data *q = qdisc_priv(sch);
	u32 band = skb->priority;
	struct tcf_result res;
	struct tcf_proto *fl;
	int err;

	*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
	if (TC_H_MAJ(skb->priority) != sch->handle) {
		fl = rcu_dereference_bh(q->filter_list);
		err = tcf_classify(skb, fl, &res, false);
#ifdef CONFIG_NET_CLS_ACT
		switch (err) {
		case TC_ACT_STOLEN:
		case TC_ACT_QUEUED:
		case TC_ACT_TRAP:
			*qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
			/* fall through */
		case TC_ACT_SHOT:
			return NULL;
		}
#endif
		if (!fl || err < 0) {
			if (TC_H_MAJ(band))
				band = 0;
			return q->queues[q->prio2band[band & TC_PRIO_MAX]];
		}
		band = res.classid;
	}
	band = TC_H_MIN(band) - 1;
	if (band >= q->bands)
		return q->queues[q->prio2band[0]];

	return q->queues[band];
}

static int
prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
{
	unsigned int len = qdisc_pkt_len(skb);
	struct Qdisc *qdisc;
	int ret;

	qdisc = prio_classify(skb, sch, &ret);
#ifdef CONFIG_NET_CLS_ACT
	if (qdisc == NULL) {

		if (ret & __NET_XMIT_BYPASS)
			qdisc_qstats_drop(sch);
		__qdisc_drop(skb, to_free);
		return ret;
	}
#endif

	ret = qdisc_enqueue(skb, qdisc, to_free);
	if (ret == NET_XMIT_SUCCESS) {
		sch->qstats.backlog += len;
		sch->q.qlen++;
		return NET_XMIT_SUCCESS;
	}
	if (net_xmit_drop_count(ret))
		qdisc_qstats_drop(sch);
	return ret;
}

static struct sk_buff *prio_peek(struct Qdisc *sch)
{
	struct prio_sched_data *q = qdisc_priv(sch);
	int prio;

	for (prio = 0; prio < q->bands; prio++) {
		struct Qdisc *qdisc = q->queues[prio];
		struct sk_buff *skb = qdisc->ops->peek(qdisc);
		if (skb)
			return skb;
	}
	return NULL;
}

static struct sk_buff *prio_dequeue(struct Qdisc *sch)
{
	struct prio_sched_data *q = qdisc_priv(sch);
	int prio;

	for (prio = 0; prio < q->bands; prio++) {
		struct Qdisc *qdisc = q->queues[prio];
		struct sk_buff *skb = qdisc_dequeue_peeked(qdisc);
		if (skb) {
			qdisc_bstats_update(sch, skb);
			qdisc_qstats_backlog_dec(sch, skb);
			sch->q.qlen--;
			return skb;
		}
	}
	return NULL;

}

static void
prio_reset(struct Qdisc *sch)
{
	int prio;
	struct prio_sched_data *q = qdisc_priv(sch);

	for (prio = 0; prio < q->bands; prio++)
		qdisc_reset(q->queues[prio]);
	sch->qstats.backlog = 0;
	sch->q.qlen = 0;
}

static int prio_offload(struct Qdisc *sch, struct tc_prio_qopt *qopt)
{
	struct net_device *dev = qdisc_dev(sch);
	struct tc_prio_qopt_offload opt = {
		.handle = sch->handle,
		.parent = sch->parent,
	};

	if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
		return -EOPNOTSUPP;

	if (qopt) {
		opt.command = TC_PRIO_REPLACE;
		opt.replace_params.bands = qopt->bands;
		memcpy(&opt.replace_params.priomap, qopt->priomap,
		       TC_PRIO_MAX + 1);
		opt.replace_params.qstats = &sch->qstats;
	} else {
		opt.command = TC_PRIO_DESTROY;
	}

	return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_PRIO, &opt);
}

static void
prio_destroy(struct Qdisc *sch)
{
	int prio;
	struct prio_sched_data *q = qdisc_priv(sch);

	tcf_block_put(q->block);
	prio_offload(sch, NULL);
	for (prio = 0; prio < q->bands; prio++)
		qdisc_put(q->queues[prio]);
}

static int prio_tune(struct Qdisc *sch, struct nlattr *opt,
		     struct netlink_ext_ack *extack)
{
	struct prio_sched_data *q = qdisc_priv(sch);
	struct Qdisc *queues[TCQ_PRIO_BANDS];
	int oldbands = q->bands, i;
	struct tc_prio_qopt *qopt;

	if (nla_len(opt) < sizeof(*qopt))
		return -EINVAL;
	qopt = nla_data(opt);

	if (qopt->bands > TCQ_PRIO_BANDS || qopt->bands < 2)
		return -EINVAL;

	for (i = 0; i <= TC_PRIO_MAX; i++) {
		if (qopt->priomap[i] >= qopt->bands)
			return -EINVAL;
	}

	/* Before commit, make sure we can allocate all new qdiscs */
	for (i = oldbands; i < qopt->bands; i++) {
		queues[i] = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
					      TC_H_MAKE(sch->handle, i + 1),
					      extack);
		if (!queues[i]) {
			while (i > oldbands)
				qdisc_put(queues[--i]);
			return -ENOMEM;
		}
	}

	prio_offload(sch, qopt);
	sch_tree_lock(sch);
	q->bands = qopt->bands;
	memcpy(q->prio2band, qopt->priomap, TC_PRIO_MAX+1);

	for (i = q->bands; i < oldbands; i++)
		qdisc_tree_flush_backlog(q->queues[i]);

	for (i = oldbands; i < q->bands; i++) {
		q->queues[i] = queues[i];
		if (q->queues[i] != &noop_qdisc)
			qdisc_hash_add(q->queues[i], true);
	}

	sch_tree_unlock(sch);

	for (i = q->bands; i < oldbands; i++)
		qdisc_put(q->queues[i]);
	return 0;
}

static int prio_init(struct Qdisc *sch, struct nlattr *opt,
		     struct netlink_ext_ack *extack)
{
	struct prio_sched_data *q = qdisc_priv(sch);
	int err;

	if (!opt)
		return -EINVAL;

	err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
	if (err)
		return err;

	return prio_tune(sch, opt, extack);
}

static int prio_dump_offload(struct Qdisc *sch)
{
	struct tc_prio_qopt_offload hw_stats = {
		.command = TC_PRIO_STATS,
		.handle = sch->handle,
		.parent = sch->parent,
		{
			.stats = {
				.bstats = &sch->bstats,
				.qstats = &sch->qstats,
			},
		},
	};

	return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_PRIO, &hw_stats);
}

static int prio_dump(struct Qdisc *sch, struct sk_buff *skb)
{
	struct prio_sched_data *q = qdisc_priv(sch);
	unsigned char *b = skb_tail_pointer(skb);
	struct tc_prio_qopt opt;
	int err;

	opt.bands = q->bands;
	memcpy(&opt.priomap, q->prio2band, TC_PRIO_MAX + 1);

	err = prio_dump_offload(sch);
	if (err)
		goto nla_put_failure;

	if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
		goto nla_put_failure;

	return skb->len;

nla_put_failure:
	nlmsg_trim(skb, b);
	return -1;
}

static int prio_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
		      struct Qdisc **old, struct netlink_ext_ack *extack)
{
	struct prio_sched_data *q = qdisc_priv(sch);
	struct tc_prio_qopt_offload graft_offload;
	unsigned long band = arg - 1;

	if (new == NULL)
		new = &noop_qdisc;

	*old = qdisc_replace(sch, new, &q->queues[band]);

	graft_offload.handle = sch->handle;
	graft_offload.parent = sch->parent;
	graft_offload.graft_params.band = band;
	graft_offload.graft_params.child_handle = new->handle;
	graft_offload.command = TC_PRIO_GRAFT;

	qdisc_offload_graft_helper(qdisc_dev(sch), sch, new, *old,
				   TC_SETUP_QDISC_PRIO, &graft_offload,
				   extack);
	return 0;
}

static struct Qdisc *
prio_leaf(struct Qdisc *sch, unsigned long arg)
{
	struct prio_sched_data *q = qdisc_priv(sch);
	unsigned long band = arg - 1;

	return q->queues[band];
}

static unsigned long prio_find(struct Qdisc *sch, u32 classid)
{
	struct prio_sched_data *q = qdisc_priv(sch);
	unsigned long band = TC_H_MIN(classid);

	if (band - 1 >= q->bands)
		return 0;
	return band;
}

static unsigned long prio_bind(struct Qdisc *sch, unsigned long parent, u32 classid)
{
	return prio_find(sch, classid);
}


static void prio_unbind(struct Qdisc *q, unsigned long cl)
{
}

static int prio_dump_class(struct Qdisc *sch, unsigned long cl, struct sk_buff *skb,
			   struct tcmsg *tcm)
{
	struct prio_sched_data *q = qdisc_priv(sch);

	tcm->tcm_handle |= TC_H_MIN(cl);
	tcm->tcm_info = q->queues[cl-1]->handle;
	return 0;
}

static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
				 struct gnet_dump *d)
{
	struct prio_sched_data *q = qdisc_priv(sch);
	struct Qdisc *cl_q;

	cl_q = q->queues[cl - 1];
	if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
				  d, NULL, &cl_q->bstats) < 0 ||
	    qdisc_qstats_copy(d, cl_q) < 0)
		return -1;

	return 0;
}

static void prio_walk(struct Qdisc *sch, struct qdisc_walker *arg)
{
	struct prio_sched_data *q = qdisc_priv(sch);
	int prio;

	if (arg->stop)
		return;

	for (prio = 0; prio < q->bands; prio++) {
		if (arg->count < arg->skip) {
			arg->count++;
			continue;
		}
		if (arg->fn(sch, prio + 1, arg) < 0) {
			arg->stop = 1;
			break;
		}
		arg->count++;
	}
}

static struct tcf_block *prio_tcf_block(struct Qdisc *sch, unsigned long cl,
					struct netlink_ext_ack *extack)
{
	struct prio_sched_data *q = qdisc_priv(sch);

	if (cl)
		return NULL;
	return q->block;
}

static const struct Qdisc_class_ops prio_class_ops = {
	.graft		=	prio_graft,
	.leaf		=	prio_leaf,
	.find		=	prio_find,
	.walk		=	prio_walk,
	.tcf_block	=	prio_tcf_block,
	.bind_tcf	=	prio_bind,
	.unbind_tcf	=	prio_unbind,
	.dump		=	prio_dump_class,
	.dump_stats	=	prio_dump_class_stats,
};

static struct Qdisc_ops prio_qdisc_ops __read_mostly = {
	.next		=	NULL,
	.cl_ops		=	&prio_class_ops,
	.id		=	"prio",
	.priv_size	=	sizeof(struct prio_sched_data),
	.enqueue	=	prio_enqueue,
	.dequeue	=	prio_dequeue,
	.peek		=	prio_peek,
	.init		=	prio_init,
	.reset		=	prio_reset,
	.destroy	=	prio_destroy,
	.change		=	prio_tune,
	.dump		=	prio_dump,
	.owner		=	THIS_MODULE,
};

static int __init prio_module_init(void)
{
	return register_qdisc(&prio_qdisc_ops);
}

static void __exit prio_module_exit(void)
{
	unregister_qdisc(&prio_qdisc_ops);
}

module_init(prio_module_init)
module_exit(prio_module_exit)

MODULE_LICENSE("GPL");
Commit	Line	Data
2874c5fd	1	// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4 LT	2	/*
	3	* net/sched/sch_prio.c Simple 3-band priority "scheduler".
	4	*
1da177e4	5	* Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
10297b99	6	* Fixes: 19990609: J Hadi Salim <hadi@nortelnetworks.com>:
1da177e4 LT	7	* Init -- EINVAL when opt undefined
	8	*/
	9
1da177e4	10	#include <linux/module.h>
5a0e3ad6	11	#include <linux/slab.h>
1da177e4 LT	12	#include <linux/types.h>
1da177e4 LT	13	#include <linux/kernel.h>
1da177e4	14	#include <linux/string.h>
1da177e4	15	#include <linux/errno.h>
1da177e4	16	#include <linux/skbuff.h>
dc5fc579	17	#include <net/netlink.h>
1da177e4	18	#include <net/pkt_sched.h>
cf1facda	19	#include <net/pkt_cls.h>
1da177e4	20
cc7ec456	21	struct prio_sched_data {
1da177e4	22	int bands;
25d8c0d5	23	struct tcf_proto __rcu *filter_list;
6529eaba	24	struct tcf_block *block;
1da177e4 LT	25	u8 prio2band[TC_PRIO_MAX+1];
	26	struct Qdisc *queues[TCQ_PRIO_BANDS];
	27	};
	28
	29
	30	static struct Qdisc *
	31	prio_classify(struct sk_buff skb, struct Qdisc sch, int *qerr)
	32	{
	33	struct prio_sched_data *q = qdisc_priv(sch);
	34	u32 band = skb->priority;
	35	struct tcf_result res;
25d8c0d5	36	struct tcf_proto *fl;
bdba91ec	37	int err;
1da177e4	38
c27f339a	39	*qerr = NET_XMIT_SUCCESS \| __NET_XMIT_BYPASS;
1da177e4	40	if (TC_H_MAJ(skb->priority) != sch->handle) {
25d8c0d5	41	fl = rcu_dereference_bh(q->filter_list);
87d83093	42	err = tcf_classify(skb, fl, &res, false);
1da177e4	43	#ifdef CONFIG_NET_CLS_ACT
dbaaa07a	44	switch (err) {
1da177e4 LT	45	case TC_ACT_STOLEN:
1da177e4 LT	46	case TC_ACT_QUEUED:
e25ea21f	47	case TC_ACT_TRAP:
378a2f09	48	*qerr = NET_XMIT_SUCCESS \| __NET_XMIT_STOLEN;
f3ae608e	49	/* fall through */
1da177e4 LT	50	case TC_ACT_SHOT:
1da177e4 LT	51	return NULL;
3ff50b79	52	}
1da177e4	53	#endif
25d8c0d5	54	if (!fl \|\| err < 0) {
1da177e4 LT	55	if (TC_H_MAJ(band))
1da177e4 LT	56	band = 0;
cc7ec456	57	return q->queues[q->prio2band[band & TC_PRIO_MAX]];
1da177e4 LT	58	}
	59	band = res.classid;
	60	}
	61	band = TC_H_MIN(band) - 1;
3e5c2d3b	62	if (band >= q->bands)
1d8ae3fd DM	63	return q->queues[q->prio2band[0]];
1d8ae3fd DM	64
1da177e4 LT	65	return q->queues[band];
	66	}
	67
	68	static int
520ac30f	69	prio_enqueue(struct sk_buff skb, struct Qdisc sch, struct sk_buff **to_free)
1da177e4	70	{
f6bab199	71	unsigned int len = qdisc_pkt_len(skb);
1da177e4 LT	72	struct Qdisc *qdisc;
	73	int ret;
	74
	75	qdisc = prio_classify(skb, sch, &ret);
	76	#ifdef CONFIG_NET_CLS_ACT
	77	if (qdisc == NULL) {
29f1df6c	78
c27f339a	79	if (ret & __NET_XMIT_BYPASS)
25331d6c	80	qdisc_qstats_drop(sch);
39ad1297	81	__qdisc_drop(skb, to_free);
1da177e4 LT	82	return ret;
	83	}
	84	#endif
	85
520ac30f	86	ret = qdisc_enqueue(skb, qdisc, to_free);
5f86173b	87	if (ret == NET_XMIT_SUCCESS) {
f6bab199	88	sch->qstats.backlog += len;
1da177e4 LT	89	sch->q.qlen++;
	90	return NET_XMIT_SUCCESS;
	91	}
378a2f09	92	if (net_xmit_drop_count(ret))
25331d6c	93	qdisc_qstats_drop(sch);
10297b99	94	return ret;
1da177e4 LT	95	}
1da177e4 LT	96
48a8f519 PM	97	static struct sk_buff prio_peek(struct Qdisc sch)
	98	{
	99	struct prio_sched_data *q = qdisc_priv(sch);
	100	int prio;
	101
	102	for (prio = 0; prio < q->bands; prio++) {
	103	struct Qdisc *qdisc = q->queues[prio];
	104	struct sk_buff *skb = qdisc->ops->peek(qdisc);
	105	if (skb)
	106	return skb;
	107	}
	108	return NULL;
	109	}
1da177e4	110
cc7ec456	111	static struct sk_buff prio_dequeue(struct Qdisc sch)
1da177e4	112	{
1da177e4 LT	113	struct prio_sched_data *q = qdisc_priv(sch);
1da177e4 LT	114	int prio;
1da177e4 LT	115
1da177e4 LT	116	for (prio = 0; prio < q->bands; prio++) {
1d8ae3fd	117	struct Qdisc *qdisc = q->queues[prio];
3557619f	118	struct sk_buff *skb = qdisc_dequeue_peeked(qdisc);
1d8ae3fd	119	if (skb) {
9190b3b3	120	qdisc_bstats_update(sch, skb);
6529d75a	121	qdisc_qstats_backlog_dec(sch, skb);
1d8ae3fd DM	122	sch->q.qlen--;
1d8ae3fd DM	123	return skb;
1da177e4 LT	124	}
	125	}
	126	return NULL;
	127
	128	}
	129
1da177e4	130	static void
cc7ec456	131	prio_reset(struct Qdisc *sch)
1da177e4 LT	132	{
	133	int prio;
	134	struct prio_sched_data *q = qdisc_priv(sch);
	135
cc7ec456	136	for (prio = 0; prio < q->bands; prio++)
1da177e4	137	qdisc_reset(q->queues[prio]);
6529d75a	138	sch->qstats.backlog = 0;
1da177e4 LT	139	sch->q.qlen = 0;
	140	}
	141
98ceb7b6	142	static int prio_offload(struct Qdisc sch, struct tc_prio_qopt qopt)
7fdb61b4	143	{
7fdb61b4 NF	144	struct net_device *dev = qdisc_dev(sch);
	145	struct tc_prio_qopt_offload opt = {
	146	.handle = sch->handle,
	147	.parent = sch->parent,
	148	};
	149
	150	if (!tc_can_offload(dev) \|\| !dev->netdev_ops->ndo_setup_tc)
	151	return -EOPNOTSUPP;
	152
98ceb7b6	153	if (qopt) {
7fdb61b4	154	opt.command = TC_PRIO_REPLACE;
98ceb7b6 NF	155	opt.replace_params.bands = qopt->bands;
98ceb7b6 NF	156	memcpy(&opt.replace_params.priomap, qopt->priomap,
7fdb61b4 NF	157	TC_PRIO_MAX + 1);
	158	opt.replace_params.qstats = &sch->qstats;
	159	} else {
	160	opt.command = TC_PRIO_DESTROY;
	161	}
	162
	163	return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_PRIO, &opt);
	164	}
	165
1da177e4	166	static void
cc7ec456	167	prio_destroy(struct Qdisc *sch)
1da177e4 LT	168	{
	169	int prio;
	170	struct prio_sched_data *q = qdisc_priv(sch);
1da177e4	171
6529eaba	172	tcf_block_put(q->block);
98ceb7b6	173	prio_offload(sch, NULL);
cc7ec456	174	for (prio = 0; prio < q->bands; prio++)
86bd446b	175	qdisc_put(q->queues[prio]);
1da177e4 LT	176	}
1da177e4 LT	177
2030721c AA	178	static int prio_tune(struct Qdisc sch, struct nlattr opt,
2030721c AA	179	struct netlink_ext_ack *extack)
1da177e4 LT	180	{
1da177e4 LT	181	struct prio_sched_data *q = qdisc_priv(sch);
3d7c8257 ED	182	struct Qdisc *queues[TCQ_PRIO_BANDS];
3d7c8257 ED	183	int oldbands = q->bands, i;
d62733c8	184	struct tc_prio_qopt *qopt;
1da177e4	185
1d8ae3fd DM	186	if (nla_len(opt) < sizeof(*qopt))
	187	return -EINVAL;
	188	qopt = nla_data(opt);
d62733c8	189
1d8ae3fd	190	if (qopt->bands > TCQ_PRIO_BANDS \|\| qopt->bands < 2)
1da177e4 LT	191	return -EINVAL;
1da177e4 LT	192
cc7ec456	193	for (i = 0; i <= TC_PRIO_MAX; i++) {
1d8ae3fd	194	if (qopt->priomap[i] >= qopt->bands)
1da177e4 LT	195	return -EINVAL;
	196	}
	197
3d7c8257 ED	198	/* Before commit, make sure we can allocate all new qdiscs */
	199	for (i = oldbands; i < qopt->bands; i++) {
	200	queues[i] = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
a38a9882 AA	201	TC_H_MAKE(sch->handle, i + 1),
a38a9882 AA	202	extack);
3d7c8257 ED	203	if (!queues[i]) {
3d7c8257 ED	204	while (i > oldbands)
86bd446b	205	qdisc_put(queues[--i]);
3d7c8257 ED	206	return -ENOMEM;
	207	}
	208	}
	209
98ceb7b6	210	prio_offload(sch, qopt);
1da177e4	211	sch_tree_lock(sch);
1d8ae3fd	212	q->bands = qopt->bands;
1da177e4 LT	213	memcpy(q->prio2band, qopt->priomap, TC_PRIO_MAX+1);
1da177e4 LT	214
e5f0e8f8 PA	215	for (i = q->bands; i < oldbands; i++)
e5f0e8f8 PA	216	qdisc_tree_flush_backlog(q->queues[i]);
cbdf4511	217
49b49971	218	for (i = oldbands; i < q->bands; i++) {
3d7c8257	219	q->queues[i] = queues[i];
49b49971 JK	220	if (q->queues[i] != &noop_qdisc)
	221	qdisc_hash_add(q->queues[i], true);
	222	}
cbdf4511	223
3d7c8257	224	sch_tree_unlock(sch);
7b8e0b6e JK	225
	226	for (i = q->bands; i < oldbands; i++)
	227	qdisc_put(q->queues[i]);
1da177e4 LT	228	return 0;
	229	}
	230
e63d7dfd AA	231	static int prio_init(struct Qdisc sch, struct nlattr opt,
e63d7dfd AA	232	struct netlink_ext_ack *extack)
1da177e4	233	{
6529eaba JP	234	struct prio_sched_data *q = qdisc_priv(sch);
	235	int err;
	236
3d7c8257	237	if (!opt)
1da177e4	238	return -EINVAL;
1da177e4	239
8d1a77f9	240	err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
6529eaba JP	241	if (err)
	242	return err;
	243
2030721c	244	return prio_tune(sch, opt, extack);
1da177e4 LT	245	}
1da177e4 LT	246
7fdb61b4 NF	247	static int prio_dump_offload(struct Qdisc *sch)
7fdb61b4 NF	248	{
7fdb61b4	249	struct tc_prio_qopt_offload hw_stats = {
ef58ca38	250	.command = TC_PRIO_STATS,
7fdb61b4 NF	251	.handle = sch->handle,
7fdb61b4 NF	252	.parent = sch->parent,
ef58ca38 AM	253	{
	254	.stats = {
	255	.bstats = &sch->bstats,
	256	.qstats = &sch->qstats,
	257	},
	258	},
7fdb61b4	259	};
7fdb61b4	260
b592843c	261	return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_PRIO, &hw_stats);
7fdb61b4 NF	262	}
7fdb61b4 NF	263
1da177e4 LT	264	static int prio_dump(struct Qdisc sch, struct sk_buff skb)
	265	{
	266	struct prio_sched_data *q = qdisc_priv(sch);
27a884dc	267	unsigned char *b = skb_tail_pointer(skb);
1da177e4	268	struct tc_prio_qopt opt;
7fdb61b4	269	int err;
1da177e4 LT	270
1da177e4 LT	271	opt.bands = q->bands;
cc7ec456	272	memcpy(&opt.priomap, q->prio2band, TC_PRIO_MAX + 1);
d62733c8	273
7fdb61b4 NF	274	err = prio_dump_offload(sch);
	275	if (err)
	276	goto nla_put_failure;
	277
1b34ec43 DM	278	if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
1b34ec43 DM	279	goto nla_put_failure;
d62733c8	280
1da177e4 LT	281	return skb->len;
1da177e4 LT	282
1e90474c	283	nla_put_failure:
dc5fc579	284	nlmsg_trim(skb, b);
1da177e4 LT	285	return -1;
	286	}
	287
	288	static int prio_graft(struct Qdisc sch, unsigned long arg, struct Qdisc new,
653d6fd6	289	struct Qdisc *old, struct netlink_ext_ack extack)
1da177e4 LT	290	{
1da177e4 LT	291	struct prio_sched_data *q = qdisc_priv(sch);
b9c7a7ac	292	struct tc_prio_qopt_offload graft_offload;
1da177e4 LT	293	unsigned long band = arg - 1;
1da177e4 LT	294
1da177e4 LT	295	if (new == NULL)
	296	new = &noop_qdisc;
	297
86a7996c	298	*old = qdisc_replace(sch, new, &q->queues[band]);
b9c7a7ac	299
b9c7a7ac NF	300	graft_offload.handle = sch->handle;
	301	graft_offload.parent = sch->parent;
	302	graft_offload.graft_params.band = band;
	303	graft_offload.graft_params.child_handle = new->handle;
	304	graft_offload.command = TC_PRIO_GRAFT;
	305
bfaee911 JK	306	qdisc_offload_graft_helper(qdisc_dev(sch), sch, new, *old,
	307	TC_SETUP_QDISC_PRIO, &graft_offload,
	308	extack);
1da177e4 LT	309	return 0;
	310	}
	311
	312	static struct Qdisc *
	313	prio_leaf(struct Qdisc *sch, unsigned long arg)
	314	{
	315	struct prio_sched_data *q = qdisc_priv(sch);
	316	unsigned long band = arg - 1;
	317
1da177e4 LT	318	return q->queues[band];
	319	}
	320
143976ce	321	static unsigned long prio_find(struct Qdisc *sch, u32 classid)
1da177e4 LT	322	{
	323	struct prio_sched_data *q = qdisc_priv(sch);
	324	unsigned long band = TC_H_MIN(classid);
	325
	326	if (band - 1 >= q->bands)
	327	return 0;
	328	return band;
	329	}
	330
	331	static unsigned long prio_bind(struct Qdisc *sch, unsigned long parent, u32 classid)
	332	{
143976ce	333	return prio_find(sch, classid);
1da177e4 LT	334	}
	335
	336
143976ce	337	static void prio_unbind(struct Qdisc *q, unsigned long cl)
1da177e4	338	{
1da177e4 LT	339	}
1da177e4 LT	340
1da177e4 LT	341	static int prio_dump_class(struct Qdisc sch, unsigned long cl, struct sk_buff skb,
	342	struct tcmsg *tcm)
	343	{
	344	struct prio_sched_data *q = qdisc_priv(sch);
	345
1da177e4	346	tcm->tcm_handle \|= TC_H_MIN(cl);
5b9a9ccf	347	tcm->tcm_info = q->queues[cl-1]->handle;
1da177e4 LT	348	return 0;
	349	}
	350
2cf6c36c JP	351	static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
	352	struct gnet_dump *d)
	353	{
	354	struct prio_sched_data *q = qdisc_priv(sch);
	355	struct Qdisc *cl_q;
	356
	357	cl_q = q->queues[cl - 1];
edb09eb1 ED	358	if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
edb09eb1 ED	359	d, NULL, &cl_q->bstats) < 0 \|\|
5dd431b6	360	qdisc_qstats_copy(d, cl_q) < 0)
2cf6c36c JP	361	return -1;
	362
	363	return 0;
	364	}
	365
1da177e4 LT	366	static void prio_walk(struct Qdisc sch, struct qdisc_walker arg)
	367	{
	368	struct prio_sched_data *q = qdisc_priv(sch);
	369	int prio;
	370
	371	if (arg->stop)
	372	return;
	373
	374	for (prio = 0; prio < q->bands; prio++) {
	375	if (arg->count < arg->skip) {
	376	arg->count++;
	377	continue;
	378	}
cc7ec456	379	if (arg->fn(sch, prio + 1, arg) < 0) {
1da177e4 LT	380	arg->stop = 1;
	381	break;
	382	}
	383	arg->count++;
	384	}
	385	}
	386
cbaacc4e AA	387	static struct tcf_block prio_tcf_block(struct Qdisc sch, unsigned long cl,
cbaacc4e AA	388	struct netlink_ext_ack *extack)
1da177e4 LT	389	{
	390	struct prio_sched_data *q = qdisc_priv(sch);
	391
	392	if (cl)
	393	return NULL;
6529eaba	394	return q->block;
1da177e4 LT	395	}
1da177e4 LT	396
20fea08b	397	static const struct Qdisc_class_ops prio_class_ops = {
1da177e4 LT	398	.graft = prio_graft,
1da177e4 LT	399	.leaf = prio_leaf,
143976ce	400	.find = prio_find,
1da177e4	401	.walk = prio_walk,
6529eaba	402	.tcf_block = prio_tcf_block,
1da177e4	403	.bind_tcf = prio_bind,
143976ce	404	.unbind_tcf = prio_unbind,
1da177e4	405	.dump = prio_dump_class,
2cf6c36c	406	.dump_stats = prio_dump_class_stats,
1da177e4 LT	407	};
1da177e4 LT	408
20fea08b	409	static struct Qdisc_ops prio_qdisc_ops __read_mostly = {
1da177e4 LT	410	.next = NULL,
	411	.cl_ops = &prio_class_ops,
	412	.id = "prio",
	413	.priv_size = sizeof(struct prio_sched_data),
	414	.enqueue = prio_enqueue,
	415	.dequeue = prio_dequeue,
48a8f519	416	.peek = prio_peek,
1da177e4 LT	417	.init = prio_init,
	418	.reset = prio_reset,
	419	.destroy = prio_destroy,
	420	.change = prio_tune,
	421	.dump = prio_dump,
	422	.owner = THIS_MODULE,
	423	};
	424
	425	static int __init prio_module_init(void)
	426	{
1d8ae3fd	427	return register_qdisc(&prio_qdisc_ops);
1da177e4 LT	428	}
1da177e4 LT	429
10297b99	430	static void __exit prio_module_exit(void)
1da177e4 LT	431	{
	432	unregister_qdisc(&prio_qdisc_ops);
	433	}
	434
	435	module_init(prio_module_init)
	436	module_exit(prio_module_exit)
	437
	438	MODULE_LICENSE("GPL");