[linux-2.6-block.git] / net / sched / act_sample.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * net/sched/act_sample.c - Packet sampling tc action
 * Copyright (c) 2017 Yotam Gigi <yotamg@mellanox.com>
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/gfp.h>
#include <net/net_namespace.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <linux/tc_act/tc_sample.h>
#include <net/tc_act/tc_sample.h>
#include <net/psample.h>
#include <net/pkt_cls.h>

#include <linux/if_arp.h>

static unsigned int sample_net_id;
static struct tc_action_ops act_sample_ops;

static const struct nla_policy sample_policy[TCA_SAMPLE_MAX + 1] = {
	[TCA_SAMPLE_PARMS]		= { .len = sizeof(struct tc_sample) },
	[TCA_SAMPLE_RATE]		= { .type = NLA_U32 },
	[TCA_SAMPLE_TRUNC_SIZE]		= { .type = NLA_U32 },
	[TCA_SAMPLE_PSAMPLE_GROUP]	= { .type = NLA_U32 },
};

static int tcf_sample_init(struct net *net, struct nlattr *nla,
			   struct nlattr *est, struct tc_action **a, int ovr,
			   int bind, bool rtnl_held, struct tcf_proto *tp,
			   struct netlink_ext_ack *extack)
{
	struct tc_action_net *tn = net_generic(net, sample_net_id);
	struct nlattr *tb[TCA_SAMPLE_MAX + 1];
	struct psample_group *psample_group;
	u32 psample_group_num, rate, index;
	struct tcf_chain *goto_ch = NULL;
	struct tc_sample *parm;
	struct tcf_sample *s;
	bool exists = false;
	int ret, err;

	if (!nla)
		return -EINVAL;
	ret = nla_parse_nested_deprecated(tb, TCA_SAMPLE_MAX, nla,
					  sample_policy, NULL);
	if (ret < 0)
		return ret;
	if (!tb[TCA_SAMPLE_PARMS] || !tb[TCA_SAMPLE_RATE] ||
	    !tb[TCA_SAMPLE_PSAMPLE_GROUP])
		return -EINVAL;

	parm = nla_data(tb[TCA_SAMPLE_PARMS]);
	index = parm->index;
	err = tcf_idr_check_alloc(tn, &index, a, bind);
	if (err < 0)
		return err;
	exists = err;
	if (exists && bind)
		return 0;

	if (!exists) {
		ret = tcf_idr_create(tn, index, est, a,
				     &act_sample_ops, bind, true);
		if (ret) {
			tcf_idr_cleanup(tn, index);
			return ret;
		}
		ret = ACT_P_CREATED;
	} else if (!ovr) {
		tcf_idr_release(*a, bind);
		return -EEXIST;
	}
	err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
	if (err < 0)
		goto release_idr;

	rate = nla_get_u32(tb[TCA_SAMPLE_RATE]);
	if (!rate) {
		NL_SET_ERR_MSG(extack, "invalid sample rate");
		err = -EINVAL;
		goto put_chain;
	}
	psample_group_num = nla_get_u32(tb[TCA_SAMPLE_PSAMPLE_GROUP]);
	psample_group = psample_group_get(net, psample_group_num);
	if (!psample_group) {
		err = -ENOMEM;
		goto put_chain;
	}

	s = to_sample(*a);

	spin_lock_bh(&s->tcf_lock);
	goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
	s->rate = rate;
	s->psample_group_num = psample_group_num;
	RCU_INIT_POINTER(s->psample_group, psample_group);

	if (tb[TCA_SAMPLE_TRUNC_SIZE]) {
		s->truncate = true;
		s->trunc_size = nla_get_u32(tb[TCA_SAMPLE_TRUNC_SIZE]);
	}
	spin_unlock_bh(&s->tcf_lock);
	if (goto_ch)
		tcf_chain_put_by_act(goto_ch);

	if (ret == ACT_P_CREATED)
		tcf_idr_insert(tn, *a);
	return ret;
put_chain:
	if (goto_ch)
		tcf_chain_put_by_act(goto_ch);
release_idr:
	tcf_idr_release(*a, bind);
	return err;
}

static void tcf_sample_cleanup(struct tc_action *a)
{
	struct tcf_sample *s = to_sample(a);
	struct psample_group *psample_group;

	/* last reference to action, no need to lock */
	psample_group = rcu_dereference_protected(s->psample_group, 1);
	RCU_INIT_POINTER(s->psample_group, NULL);
	if (psample_group)
		psample_group_put(psample_group);
}

static bool tcf_sample_dev_ok_push(struct net_device *dev)
{
	switch (dev->type) {
	case ARPHRD_TUNNEL:
	case ARPHRD_TUNNEL6:
	case ARPHRD_SIT:
	case ARPHRD_IPGRE:
	case ARPHRD_VOID:
	case ARPHRD_NONE:
		return false;
	default:
		return true;
	}
}

static int tcf_sample_act(struct sk_buff *skb, const struct tc_action *a,
			  struct tcf_result *res)
{
	struct tcf_sample *s = to_sample(a);
	struct psample_group *psample_group;
	int retval;
	int size;
	int iif;
	int oif;

	tcf_lastuse_update(&s->tcf_tm);
	bstats_cpu_update(this_cpu_ptr(s->common.cpu_bstats), skb);
	retval = READ_ONCE(s->tcf_action);

	psample_group = rcu_dereference_bh(s->psample_group);

	/* randomly sample packets according to rate */
	if (psample_group && (prandom_u32() % s->rate == 0)) {
		if (!skb_at_tc_ingress(skb)) {
			iif = skb->skb_iif;
			oif = skb->dev->ifindex;
		} else {
			iif = skb->dev->ifindex;
			oif = 0;
		}

		/* on ingress, the mac header gets popped, so push it back */
		if (skb_at_tc_ingress(skb) && tcf_sample_dev_ok_push(skb->dev))
			skb_push(skb, skb->mac_len);

		size = s->truncate ? s->trunc_size : skb->len;
		psample_sample_packet(psample_group, skb, size, iif, oif,
				      s->rate);

		if (skb_at_tc_ingress(skb) && tcf_sample_dev_ok_push(skb->dev))
			skb_pull(skb, skb->mac_len);
	}

	return retval;
}

static int tcf_sample_dump(struct sk_buff *skb, struct tc_action *a,
			   int bind, int ref)
{
	unsigned char *b = skb_tail_pointer(skb);
	struct tcf_sample *s = to_sample(a);
	struct tc_sample opt = {
		.index      = s->tcf_index,
		.refcnt     = refcount_read(&s->tcf_refcnt) - ref,
		.bindcnt    = atomic_read(&s->tcf_bindcnt) - bind,
	};
	struct tcf_t t;

	spin_lock_bh(&s->tcf_lock);
	opt.action = s->tcf_action;
	if (nla_put(skb, TCA_SAMPLE_PARMS, sizeof(opt), &opt))
		goto nla_put_failure;

	tcf_tm_dump(&t, &s->tcf_tm);
	if (nla_put_64bit(skb, TCA_SAMPLE_TM, sizeof(t), &t, TCA_SAMPLE_PAD))
		goto nla_put_failure;

	if (nla_put_u32(skb, TCA_SAMPLE_RATE, s->rate))
		goto nla_put_failure;

	if (s->truncate)
		if (nla_put_u32(skb, TCA_SAMPLE_TRUNC_SIZE, s->trunc_size))
			goto nla_put_failure;

	if (nla_put_u32(skb, TCA_SAMPLE_PSAMPLE_GROUP, s->psample_group_num))
		goto nla_put_failure;
	spin_unlock_bh(&s->tcf_lock);

	return skb->len;

nla_put_failure:
	spin_unlock_bh(&s->tcf_lock);
	nlmsg_trim(skb, b);
	return -1;
}

static int tcf_sample_walker(struct net *net, struct sk_buff *skb,
			     struct netlink_callback *cb, int type,
			     const struct tc_action_ops *ops,
			     struct netlink_ext_ack *extack)
{
	struct tc_action_net *tn = net_generic(net, sample_net_id);

	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
}

static int tcf_sample_search(struct net *net, struct tc_action **a, u32 index)
{
	struct tc_action_net *tn = net_generic(net, sample_net_id);

	return tcf_idr_search(tn, a, index);
}

static struct tc_action_ops act_sample_ops = {
	.kind	  = "sample",
	.id	  = TCA_ID_SAMPLE,
	.owner	  = THIS_MODULE,
	.act	  = tcf_sample_act,
	.dump	  = tcf_sample_dump,
	.init	  = tcf_sample_init,
	.cleanup  = tcf_sample_cleanup,
	.walk	  = tcf_sample_walker,
	.lookup	  = tcf_sample_search,
	.size	  = sizeof(struct tcf_sample),
};

static __net_init int sample_init_net(struct net *net)
{
	struct tc_action_net *tn = net_generic(net, sample_net_id);

	return tc_action_net_init(tn, &act_sample_ops);
}

static void __net_exit sample_exit_net(struct list_head *net_list)
{
	tc_action_net_exit(net_list, sample_net_id);
}

static struct pernet_operations sample_net_ops = {
	.init = sample_init_net,
	.exit_batch = sample_exit_net,
	.id   = &sample_net_id,
	.size = sizeof(struct tc_action_net),
};

static int __init sample_init_module(void)
{
	return tcf_register_action(&act_sample_ops, &sample_net_ops);
}

static void __exit sample_cleanup_module(void)
{
	tcf_unregister_action(&act_sample_ops, &sample_net_ops);
}

module_init(sample_init_module);
module_exit(sample_cleanup_module);

MODULE_AUTHOR("Yotam Gigi <yotam.gi@gmail.com>");
MODULE_DESCRIPTION("Packet sampling action");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
d2912cb1	1	// SPDX-License-Identifier: GPL-2.0-only
5c5670fa YG	2	/*
	3	* net/sched/act_sample.c - Packet sampling tc action
	4	* Copyright (c) 2017 Yotam Gigi <yotamg@mellanox.com>
5c5670fa YG	5	*/
	6
	7	#include <linux/types.h>
	8	#include <linux/kernel.h>
	9	#include <linux/string.h>
	10	#include <linux/errno.h>
	11	#include <linux/skbuff.h>
	12	#include <linux/rtnetlink.h>
	13	#include <linux/module.h>
	14	#include <linux/init.h>
	15	#include <linux/gfp.h>
	16	#include <net/net_namespace.h>
	17	#include <net/netlink.h>
	18	#include <net/pkt_sched.h>
	19	#include <linux/tc_act/tc_sample.h>
	20	#include <net/tc_act/tc_sample.h>
	21	#include <net/psample.h>
e8c87c64	22	#include <net/pkt_cls.h>
5c5670fa YG	23
	24	#include <linux/if_arp.h>
	25
5c5670fa YG	26	static unsigned int sample_net_id;
	27	static struct tc_action_ops act_sample_ops;
	28
	29	static const struct nla_policy sample_policy[TCA_SAMPLE_MAX + 1] = {
	30	[TCA_SAMPLE_PARMS] = { .len = sizeof(struct tc_sample) },
	31	[TCA_SAMPLE_RATE] = { .type = NLA_U32 },
	32	[TCA_SAMPLE_TRUNC_SIZE] = { .type = NLA_U32 },
	33	[TCA_SAMPLE_PSAMPLE_GROUP] = { .type = NLA_U32 },
	34	};
	35
	36	static int tcf_sample_init(struct net net, struct nlattr nla,
	37	struct nlattr est, struct tc_action *a, int ovr,
85d0966f	38	int bind, bool rtnl_held, struct tcf_proto *tp,
789871bb	39	struct netlink_ext_ack *extack)
5c5670fa YG	40	{
	41	struct tc_action_net *tn = net_generic(net, sample_net_id);
	42	struct nlattr *tb[TCA_SAMPLE_MAX + 1];
	43	struct psample_group *psample_group;
7be8ef2c	44	u32 psample_group_num, rate, index;
e8c87c64	45	struct tcf_chain *goto_ch = NULL;
5c5670fa YG	46	struct tc_sample *parm;
	47	struct tcf_sample *s;
	48	bool exists = false;
0190c1d4	49	int ret, err;
5c5670fa YG	50
	51	if (!nla)
	52	return -EINVAL;
8cb08174 JB	53	ret = nla_parse_nested_deprecated(tb, TCA_SAMPLE_MAX, nla,
8cb08174 JB	54	sample_policy, NULL);
5c5670fa YG	55	if (ret < 0)
	56	return ret;
	57	if (!tb[TCA_SAMPLE_PARMS] \|\| !tb[TCA_SAMPLE_RATE] \|\|
	58	!tb[TCA_SAMPLE_PSAMPLE_GROUP])
	59	return -EINVAL;
	60
	61	parm = nla_data(tb[TCA_SAMPLE_PARMS]);
7be8ef2c DL	62	index = parm->index;
7be8ef2c DL	63	err = tcf_idr_check_alloc(tn, &index, a, bind);
0190c1d4 VB	64	if (err < 0)
	65	return err;
	66	exists = err;
5c5670fa YG	67	if (exists && bind)
	68	return 0;
	69
	70	if (!exists) {
7be8ef2c	71	ret = tcf_idr_create(tn, index, est, a,
34043d25	72	&act_sample_ops, bind, true);
0190c1d4	73	if (ret) {
7be8ef2c	74	tcf_idr_cleanup(tn, index);
5c5670fa	75	return ret;
0190c1d4	76	}
5c5670fa	77	ret = ACT_P_CREATED;
4e8ddd7f	78	} else if (!ovr) {
65a206c0	79	tcf_idr_release(*a, bind);
4e8ddd7f	80	return -EEXIST;
5c5670fa	81	}
e8c87c64 DC	82	err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
	83	if (err < 0)
	84	goto release_idr;
5c5670fa	85
fae27081 DC	86	rate = nla_get_u32(tb[TCA_SAMPLE_RATE]);
	87	if (!rate) {
	88	NL_SET_ERR_MSG(extack, "invalid sample rate");
	89	err = -EINVAL;
	90	goto put_chain;
	91	}
653cd284 VB	92	psample_group_num = nla_get_u32(tb[TCA_SAMPLE_PSAMPLE_GROUP]);
653cd284 VB	93	psample_group = psample_group_get(net, psample_group_num);
cadb9c9f	94	if (!psample_group) {
e8c87c64 DC	95	err = -ENOMEM;
e8c87c64 DC	96	goto put_chain;
cadb9c9f	97	}
653cd284 VB	98
	99	s = to_sample(*a);
	100
	101	spin_lock_bh(&s->tcf_lock);
e8c87c64	102	goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
fae27081	103	s->rate = rate;
653cd284	104	s->psample_group_num = psample_group_num;
5c5670fa YG	105	RCU_INIT_POINTER(s->psample_group, psample_group);
	106
	107	if (tb[TCA_SAMPLE_TRUNC_SIZE]) {
	108	s->truncate = true;
	109	s->trunc_size = nla_get_u32(tb[TCA_SAMPLE_TRUNC_SIZE]);
	110	}
653cd284	111	spin_unlock_bh(&s->tcf_lock);
e8c87c64 DC	112	if (goto_ch)
e8c87c64 DC	113	tcf_chain_put_by_act(goto_ch);
5c5670fa YG	114
5c5670fa YG	115	if (ret == ACT_P_CREATED)
65a206c0	116	tcf_idr_insert(tn, *a);
5c5670fa	117	return ret;
e8c87c64 DC	118	put_chain:
	119	if (goto_ch)
	120	tcf_chain_put_by_act(goto_ch);
	121	release_idr:
	122	tcf_idr_release(*a, bind);
	123	return err;
5c5670fa YG	124	}
5c5670fa YG	125
9a63b255	126	static void tcf_sample_cleanup(struct tc_action *a)
5c5670fa	127	{
90a6ec85	128	struct tcf_sample *s = to_sample(a);
5c5670fa YG	129	struct psample_group *psample_group;
5c5670fa YG	130
d7728495 VB	131	/* last reference to action, no need to lock */
d7728495 VB	132	psample_group = rcu_dereference_protected(s->psample_group, 1);
5c5670fa	133	RCU_INIT_POINTER(s->psample_group, NULL);
1f110e7c DC	134	if (psample_group)
1f110e7c DC	135	psample_group_put(psample_group);
5c5670fa YG	136	}
5c5670fa YG	137
5c5670fa YG	138	static bool tcf_sample_dev_ok_push(struct net_device *dev)
	139	{
	140	switch (dev->type) {
	141	case ARPHRD_TUNNEL:
	142	case ARPHRD_TUNNEL6:
	143	case ARPHRD_SIT:
	144	case ARPHRD_IPGRE:
	145	case ARPHRD_VOID:
	146	case ARPHRD_NONE:
	147	return false;
	148	default:
	149	return true;
	150	}
	151	}
	152
	153	static int tcf_sample_act(struct sk_buff skb, const struct tc_action a,
	154	struct tcf_result *res)
	155	{
	156	struct tcf_sample *s = to_sample(a);
	157	struct psample_group *psample_group;
	158	int retval;
	159	int size;
	160	int iif;
	161	int oif;
	162
	163	tcf_lastuse_update(&s->tcf_tm);
	164	bstats_cpu_update(this_cpu_ptr(s->common.cpu_bstats), skb);
	165	retval = READ_ONCE(s->tcf_action);
	166
7fd4b288	167	psample_group = rcu_dereference_bh(s->psample_group);
5c5670fa YG	168
	169	/* randomly sample packets according to rate */
	170	if (psample_group && (prandom_u32() % s->rate == 0)) {
	171	if (!skb_at_tc_ingress(skb)) {
	172	iif = skb->skb_iif;
	173	oif = skb->dev->ifindex;
	174	} else {
	175	iif = skb->dev->ifindex;
	176	oif = 0;
	177	}
	178
	179	/* on ingress, the mac header gets popped, so push it back */
	180	if (skb_at_tc_ingress(skb) && tcf_sample_dev_ok_push(skb->dev))
	181	skb_push(skb, skb->mac_len);
	182
	183	size = s->truncate ? s->trunc_size : skb->len;
	184	psample_sample_packet(psample_group, skb, size, iif, oif,
	185	s->rate);
	186
	187	if (skb_at_tc_ingress(skb) && tcf_sample_dev_ok_push(skb->dev))
	188	skb_pull(skb, skb->mac_len);
	189	}
	190
5c5670fa YG	191	return retval;
	192	}
	193
	194	static int tcf_sample_dump(struct sk_buff skb, struct tc_action a,
	195	int bind, int ref)
	196	{
	197	unsigned char *b = skb_tail_pointer(skb);
	198	struct tcf_sample *s = to_sample(a);
	199	struct tc_sample opt = {
	200	.index = s->tcf_index,
036bb443 VB	201	.refcnt = refcount_read(&s->tcf_refcnt) - ref,
036bb443 VB	202	.bindcnt = atomic_read(&s->tcf_bindcnt) - bind,
5c5670fa YG	203	};
	204	struct tcf_t t;
	205
653cd284	206	spin_lock_bh(&s->tcf_lock);
d7728495	207	opt.action = s->tcf_action;
5c5670fa YG	208	if (nla_put(skb, TCA_SAMPLE_PARMS, sizeof(opt), &opt))
	209	goto nla_put_failure;
	210
	211	tcf_tm_dump(&t, &s->tcf_tm);
	212	if (nla_put_64bit(skb, TCA_SAMPLE_TM, sizeof(t), &t, TCA_SAMPLE_PAD))
	213	goto nla_put_failure;
	214
	215	if (nla_put_u32(skb, TCA_SAMPLE_RATE, s->rate))
	216	goto nla_put_failure;
	217
	218	if (s->truncate)
	219	if (nla_put_u32(skb, TCA_SAMPLE_TRUNC_SIZE, s->trunc_size))
	220	goto nla_put_failure;
	221
	222	if (nla_put_u32(skb, TCA_SAMPLE_PSAMPLE_GROUP, s->psample_group_num))
	223	goto nla_put_failure;
653cd284	224	spin_unlock_bh(&s->tcf_lock);
d7728495	225
5c5670fa YG	226	return skb->len;
	227
	228	nla_put_failure:
653cd284	229	spin_unlock_bh(&s->tcf_lock);
5c5670fa YG	230	nlmsg_trim(skb, b);
	231	return -1;
	232	}
	233
	234	static int tcf_sample_walker(struct net net, struct sk_buff skb,
	235	struct netlink_callback *cb, int type,
41780105 AA	236	const struct tc_action_ops *ops,
41780105 AA	237	struct netlink_ext_ack *extack)
5c5670fa YG	238	{
	239	struct tc_action_net *tn = net_generic(net, sample_net_id);
	240
b3620145	241	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
5c5670fa YG	242	}
5c5670fa YG	243
f061b48c	244	static int tcf_sample_search(struct net net, struct tc_action *a, u32 index)
5c5670fa YG	245	{
	246	struct tc_action_net *tn = net_generic(net, sample_net_id);
	247
65a206c0	248	return tcf_idr_search(tn, a, index);
5c5670fa YG	249	}
	250
	251	static struct tc_action_ops act_sample_ops = {
	252	.kind = "sample",
eddd2cf1	253	.id = TCA_ID_SAMPLE,
5c5670fa YG	254	.owner = THIS_MODULE,
	255	.act = tcf_sample_act,
	256	.dump = tcf_sample_dump,
	257	.init = tcf_sample_init,
	258	.cleanup = tcf_sample_cleanup,
	259	.walk = tcf_sample_walker,
	260	.lookup = tcf_sample_search,
	261	.size = sizeof(struct tcf_sample),
	262	};
	263
	264	static __net_init int sample_init_net(struct net *net)
	265	{
	266	struct tc_action_net *tn = net_generic(net, sample_net_id);
	267
c7e460ce	268	return tc_action_net_init(tn, &act_sample_ops);
5c5670fa YG	269	}
5c5670fa YG	270
039af9c6	271	static void __net_exit sample_exit_net(struct list_head *net_list)
5c5670fa	272	{
039af9c6	273	tc_action_net_exit(net_list, sample_net_id);
5c5670fa YG	274	}
	275
	276	static struct pernet_operations sample_net_ops = {
	277	.init = sample_init_net,
039af9c6	278	.exit_batch = sample_exit_net,
5c5670fa YG	279	.id = &sample_net_id,
	280	.size = sizeof(struct tc_action_net),
	281	};
	282
	283	static int __init sample_init_module(void)
	284	{
	285	return tcf_register_action(&act_sample_ops, &sample_net_ops);
	286	}
	287
	288	static void __exit sample_cleanup_module(void)
	289	{
	290	tcf_unregister_action(&act_sample_ops, &sample_net_ops);
	291	}
	292
	293	module_init(sample_init_module);
	294	module_exit(sample_cleanup_module);
	295
f1fd20c3	296	MODULE_AUTHOR("Yotam Gigi <yotam.gi@gmail.com>");
5c5670fa YG	297	MODULE_DESCRIPTION("Packet sampling action");
5c5670fa YG	298	MODULE_LICENSE("GPL v2");