[linux-block.git] / include / net / tc_wrapper.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_TC_WRAPPER_H
#define __NET_TC_WRAPPER_H

#include <net/pkt_cls.h>

#if IS_ENABLED(CONFIG_RETPOLINE)

#include <linux/cpufeature.h>
#include <linux/static_key.h>
#include <linux/indirect_call_wrapper.h>

#define TC_INDIRECT_SCOPE

extern struct static_key_false tc_skip_wrapper;

/* TC Actions */
#ifdef CONFIG_NET_CLS_ACT

#define TC_INDIRECT_ACTION_DECLARE(fname)                              \
	INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb,       \
					    const struct tc_action *a, \
					    struct tcf_result *res))

TC_INDIRECT_ACTION_DECLARE(tcf_bpf_act);
TC_INDIRECT_ACTION_DECLARE(tcf_connmark_act);
TC_INDIRECT_ACTION_DECLARE(tcf_csum_act);
TC_INDIRECT_ACTION_DECLARE(tcf_ct_act);
TC_INDIRECT_ACTION_DECLARE(tcf_ctinfo_act);
TC_INDIRECT_ACTION_DECLARE(tcf_gact_act);
TC_INDIRECT_ACTION_DECLARE(tcf_gate_act);
TC_INDIRECT_ACTION_DECLARE(tcf_ife_act);
TC_INDIRECT_ACTION_DECLARE(tcf_ipt_act);
TC_INDIRECT_ACTION_DECLARE(tcf_mirred_act);
TC_INDIRECT_ACTION_DECLARE(tcf_mpls_act);
TC_INDIRECT_ACTION_DECLARE(tcf_nat_act);
TC_INDIRECT_ACTION_DECLARE(tcf_pedit_act);
TC_INDIRECT_ACTION_DECLARE(tcf_police_act);
TC_INDIRECT_ACTION_DECLARE(tcf_sample_act);
TC_INDIRECT_ACTION_DECLARE(tcf_simp_act);
TC_INDIRECT_ACTION_DECLARE(tcf_skbedit_act);
TC_INDIRECT_ACTION_DECLARE(tcf_skbmod_act);
TC_INDIRECT_ACTION_DECLARE(tcf_vlan_act);
TC_INDIRECT_ACTION_DECLARE(tunnel_key_act);

static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
			   struct tcf_result *res)
{
	if (static_branch_likely(&tc_skip_wrapper))
		goto skip;

#if IS_BUILTIN(CONFIG_NET_ACT_GACT)
	if (a->ops->act == tcf_gact_act)
		return tcf_gact_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_MIRRED)
	if (a->ops->act == tcf_mirred_act)
		return tcf_mirred_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_PEDIT)
	if (a->ops->act == tcf_pedit_act)
		return tcf_pedit_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT)
	if (a->ops->act == tcf_skbedit_act)
		return tcf_skbedit_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_SKBMOD)
	if (a->ops->act == tcf_skbmod_act)
		return tcf_skbmod_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_POLICE)
	if (a->ops->act == tcf_police_act)
		return tcf_police_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_BPF)
	if (a->ops->act == tcf_bpf_act)
		return tcf_bpf_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_CONNMARK)
	if (a->ops->act == tcf_connmark_act)
		return tcf_connmark_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_CSUM)
	if (a->ops->act == tcf_csum_act)
		return tcf_csum_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_CT)
	if (a->ops->act == tcf_ct_act)
		return tcf_ct_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_CTINFO)
	if (a->ops->act == tcf_ctinfo_act)
		return tcf_ctinfo_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_GATE)
	if (a->ops->act == tcf_gate_act)
		return tcf_gate_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_MPLS)
	if (a->ops->act == tcf_mpls_act)
		return tcf_mpls_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_NAT)
	if (a->ops->act == tcf_nat_act)
		return tcf_nat_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY)
	if (a->ops->act == tunnel_key_act)
		return tunnel_key_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_VLAN)
	if (a->ops->act == tcf_vlan_act)
		return tcf_vlan_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_IFE)
	if (a->ops->act == tcf_ife_act)
		return tcf_ife_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_SIMP)
	if (a->ops->act == tcf_simp_act)
		return tcf_simp_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_SAMPLE)
	if (a->ops->act == tcf_sample_act)
		return tcf_sample_act(skb, a, res);
#endif

skip:
	return a->ops->act(skb, a, res);
}

#endif /* CONFIG_NET_CLS_ACT */

/* TC Filters */
#ifdef CONFIG_NET_CLS

#define TC_INDIRECT_FILTER_DECLARE(fname)                               \
	INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb,        \
					    const struct tcf_proto *tp, \
					    struct tcf_result *res))

TC_INDIRECT_FILTER_DECLARE(basic_classify);
TC_INDIRECT_FILTER_DECLARE(cls_bpf_classify);
TC_INDIRECT_FILTER_DECLARE(cls_cgroup_classify);
TC_INDIRECT_FILTER_DECLARE(fl_classify);
TC_INDIRECT_FILTER_DECLARE(flow_classify);
TC_INDIRECT_FILTER_DECLARE(fw_classify);
TC_INDIRECT_FILTER_DECLARE(mall_classify);
TC_INDIRECT_FILTER_DECLARE(route4_classify);
TC_INDIRECT_FILTER_DECLARE(u32_classify);

static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
				struct tcf_result *res)
{
	if (static_branch_likely(&tc_skip_wrapper))
		goto skip;

#if IS_BUILTIN(CONFIG_NET_CLS_BPF)
	if (tp->classify == cls_bpf_classify)
		return cls_bpf_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_U32)
	if (tp->classify == u32_classify)
		return u32_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_FLOWER)
	if (tp->classify == fl_classify)
		return fl_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_FW)
	if (tp->classify == fw_classify)
		return fw_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_MATCHALL)
	if (tp->classify == mall_classify)
		return mall_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_BASIC)
	if (tp->classify == basic_classify)
		return basic_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_CGROUP)
	if (tp->classify == cls_cgroup_classify)
		return cls_cgroup_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_FLOW)
	if (tp->classify == flow_classify)
		return flow_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_ROUTE4)
	if (tp->classify == route4_classify)
		return route4_classify(skb, tp, res);
#endif

skip:
	return tp->classify(skb, tp, res);
}

#endif /* CONFIG_NET_CLS */

static inline void tc_wrapper_init(void)
{
#ifdef CONFIG_X86
	if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE))
		static_branch_enable(&tc_skip_wrapper);
#endif
}

#else

#define TC_INDIRECT_SCOPE static

static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
			   struct tcf_result *res)
{
	return a->ops->act(skb, a, res);
}

static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
				struct tcf_result *res)
{
	return tp->classify(skb, tp, res);
}

static inline void tc_wrapper_init(void)
{
}

#endif

#endif /* __NET_TC_WRAPPER_H */
Commit	Line	Data
7f0e8102 PT	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef __NET_TC_WRAPPER_H
	3	#define __NET_TC_WRAPPER_H
	4
	5	#include <net/pkt_cls.h>
	6
	7	#if IS_ENABLED(CONFIG_RETPOLINE)
	8
	9	#include <linux/cpufeature.h>
	10	#include <linux/static_key.h>
	11	#include <linux/indirect_call_wrapper.h>
	12
	13	#define TC_INDIRECT_SCOPE
	14
	15	extern struct static_key_false tc_skip_wrapper;
	16
	17	/* TC Actions */
	18	#ifdef CONFIG_NET_CLS_ACT
	19
	20	#define TC_INDIRECT_ACTION_DECLARE(fname) \
	21	INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
	22	const struct tc_action *a, \
	23	struct tcf_result *res))
	24
	25	TC_INDIRECT_ACTION_DECLARE(tcf_bpf_act);
	26	TC_INDIRECT_ACTION_DECLARE(tcf_connmark_act);
	27	TC_INDIRECT_ACTION_DECLARE(tcf_csum_act);
	28	TC_INDIRECT_ACTION_DECLARE(tcf_ct_act);
	29	TC_INDIRECT_ACTION_DECLARE(tcf_ctinfo_act);
	30	TC_INDIRECT_ACTION_DECLARE(tcf_gact_act);
	31	TC_INDIRECT_ACTION_DECLARE(tcf_gate_act);
	32	TC_INDIRECT_ACTION_DECLARE(tcf_ife_act);
	33	TC_INDIRECT_ACTION_DECLARE(tcf_ipt_act);
	34	TC_INDIRECT_ACTION_DECLARE(tcf_mirred_act);
	35	TC_INDIRECT_ACTION_DECLARE(tcf_mpls_act);
	36	TC_INDIRECT_ACTION_DECLARE(tcf_nat_act);
	37	TC_INDIRECT_ACTION_DECLARE(tcf_pedit_act);
	38	TC_INDIRECT_ACTION_DECLARE(tcf_police_act);
	39	TC_INDIRECT_ACTION_DECLARE(tcf_sample_act);
	40	TC_INDIRECT_ACTION_DECLARE(tcf_simp_act);
	41	TC_INDIRECT_ACTION_DECLARE(tcf_skbedit_act);
	42	TC_INDIRECT_ACTION_DECLARE(tcf_skbmod_act);
	43	TC_INDIRECT_ACTION_DECLARE(tcf_vlan_act);
	44	TC_INDIRECT_ACTION_DECLARE(tunnel_key_act);
	45
	46	static inline int tc_act(struct sk_buff skb, const struct tc_action a,
	47	struct tcf_result *res)
	48	{
	49	if (static_branch_likely(&tc_skip_wrapper))
	50	goto skip;
	51
	52	#if IS_BUILTIN(CONFIG_NET_ACT_GACT)
	53	if (a->ops->act == tcf_gact_act)
	54	return tcf_gact_act(skb, a, res);
	55	#endif
	56	#if IS_BUILTIN(CONFIG_NET_ACT_MIRRED)
	57	if (a->ops->act == tcf_mirred_act)
	58	return tcf_mirred_act(skb, a, res);
	59	#endif
	60	#if IS_BUILTIN(CONFIG_NET_ACT_PEDIT)
	61	if (a->ops->act == tcf_pedit_act)
	62	return tcf_pedit_act(skb, a, res);
	63	#endif
	64	#if IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT)
65	if (a->ops->act == tcf_skbedit_act)
66	return tcf_skbedit_act(skb, a, res);
67	#endif
68	#if IS_BUILTIN(CONFIG_NET_ACT_SKBMOD)
69	if (a->ops->act == tcf_skbmod_act)
70	return tcf_skbmod_act(skb, a, res);
71	#endif
72	#if IS_BUILTIN(CONFIG_NET_ACT_POLICE)
73	if (a->ops->act == tcf_police_act)
74	return tcf_police_act(skb, a, res);
75	#endif
76	#if IS_BUILTIN(CONFIG_NET_ACT_BPF)
77	if (a->ops->act == tcf_bpf_act)
78	return tcf_bpf_act(skb, a, res);
79	#endif
80	#if IS_BUILTIN(CONFIG_NET_ACT_CONNMARK)
81	if (a->ops->act == tcf_connmark_act)
82	return tcf_connmark_act(skb, a, res);
83	#endif
84	#if IS_BUILTIN(CONFIG_NET_ACT_CSUM)
85	if (a->ops->act == tcf_csum_act)
86	return tcf_csum_act(skb, a, res);
87	#endif
88	#if IS_BUILTIN(CONFIG_NET_ACT_CT)
89	if (a->ops->act == tcf_ct_act)
90	return tcf_ct_act(skb, a, res);
91	#endif
92	#if IS_BUILTIN(CONFIG_NET_ACT_CTINFO)
93	if (a->ops->act == tcf_ctinfo_act)
94	return tcf_ctinfo_act(skb, a, res);
95	#endif
96	#if IS_BUILTIN(CONFIG_NET_ACT_GATE)
97	if (a->ops->act == tcf_gate_act)
98	return tcf_gate_act(skb, a, res);
99	#endif
100	#if IS_BUILTIN(CONFIG_NET_ACT_MPLS)
101	if (a->ops->act == tcf_mpls_act)
102	return tcf_mpls_act(skb, a, res);
103	#endif
104	#if IS_BUILTIN(CONFIG_NET_ACT_NAT)
105	if (a->ops->act == tcf_nat_act)
106	return tcf_nat_act(skb, a, res);
107	#endif
108	#if IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY)
109	if (a->ops->act == tunnel_key_act)
110	return tunnel_key_act(skb, a, res);
111	#endif
112	#if IS_BUILTIN(CONFIG_NET_ACT_VLAN)
113	if (a->ops->act == tcf_vlan_act)
114	return tcf_vlan_act(skb, a, res);
115	#endif
116	#if IS_BUILTIN(CONFIG_NET_ACT_IFE)
117	if (a->ops->act == tcf_ife_act)
118	return tcf_ife_act(skb, a, res);
119	#endif
7f0e8102 PT	120	#if IS_BUILTIN(CONFIG_NET_ACT_SIMP)
	121	if (a->ops->act == tcf_simp_act)
	122	return tcf_simp_act(skb, a, res);
	123	#endif
	124	#if IS_BUILTIN(CONFIG_NET_ACT_SAMPLE)
	125	if (a->ops->act == tcf_sample_act)
	126	return tcf_sample_act(skb, a, res);
	127	#endif
	128
	129	skip:
	130	return a->ops->act(skb, a, res);
	131	}
	132
	133	#endif /* CONFIG_NET_CLS_ACT */
	134
	135	/* TC Filters */
	136	#ifdef CONFIG_NET_CLS
	137
	138	#define TC_INDIRECT_FILTER_DECLARE(fname) \
	139	INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
	140	const struct tcf_proto *tp, \
	141	struct tcf_result *res))
	142
	143	TC_INDIRECT_FILTER_DECLARE(basic_classify);
	144	TC_INDIRECT_FILTER_DECLARE(cls_bpf_classify);
	145	TC_INDIRECT_FILTER_DECLARE(cls_cgroup_classify);
	146	TC_INDIRECT_FILTER_DECLARE(fl_classify);
	147	TC_INDIRECT_FILTER_DECLARE(flow_classify);
	148	TC_INDIRECT_FILTER_DECLARE(fw_classify);
	149	TC_INDIRECT_FILTER_DECLARE(mall_classify);
	150	TC_INDIRECT_FILTER_DECLARE(route4_classify);
7f0e8102 PT	151	TC_INDIRECT_FILTER_DECLARE(u32_classify);
	152
	153	static inline int tc_classify(struct sk_buff skb, const struct tcf_proto tp,
	154	struct tcf_result *res)
	155	{
	156	if (static_branch_likely(&tc_skip_wrapper))
	157	goto skip;
	158
	159	#if IS_BUILTIN(CONFIG_NET_CLS_BPF)
	160	if (tp->classify == cls_bpf_classify)
	161	return cls_bpf_classify(skb, tp, res);
	162	#endif
	163	#if IS_BUILTIN(CONFIG_NET_CLS_U32)
	164	if (tp->classify == u32_classify)
	165	return u32_classify(skb, tp, res);
	166	#endif
	167	#if IS_BUILTIN(CONFIG_NET_CLS_FLOWER)
	168	if (tp->classify == fl_classify)
	169	return fl_classify(skb, tp, res);
	170	#endif
	171	#if IS_BUILTIN(CONFIG_NET_CLS_FW)
	172	if (tp->classify == fw_classify)
	173	return fw_classify(skb, tp, res);
	174	#endif
	175	#if IS_BUILTIN(CONFIG_NET_CLS_MATCHALL)
	176	if (tp->classify == mall_classify)
	177	return mall_classify(skb, tp, res);
	178	#endif
	179	#if IS_BUILTIN(CONFIG_NET_CLS_BASIC)
	180	if (tp->classify == basic_classify)
	181	return basic_classify(skb, tp, res);
	182	#endif
	183	#if IS_BUILTIN(CONFIG_NET_CLS_CGROUP)
	184	if (tp->classify == cls_cgroup_classify)
	185	return cls_cgroup_classify(skb, tp, res);
	186	#endif
	187	#if IS_BUILTIN(CONFIG_NET_CLS_FLOW)
	188	if (tp->classify == flow_classify)
	189	return flow_classify(skb, tp, res);
	190	#endif
	191	#if IS_BUILTIN(CONFIG_NET_CLS_ROUTE4)
	192	if (tp->classify == route4_classify)
	193	return route4_classify(skb, tp, res);
	194	#endif
7f0e8102 PT	195
	196	skip:
	197	return tp->classify(skb, tp, res);
	198	}
	199
40cab44b PT	200	#endif /* CONFIG_NET_CLS */
40cab44b PT	201
7f0e8102 PT	202	static inline void tc_wrapper_init(void)
	203	{
	204	#ifdef CONFIG_X86
	205	if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE))
	206	static_branch_enable(&tc_skip_wrapper);
	207	#endif
	208	}
	209
7f0e8102 PT	210	#else
	211
	212	#define TC_INDIRECT_SCOPE static
	213
	214	static inline int tc_act(struct sk_buff skb, const struct tc_action a,
	215	struct tcf_result *res)
	216	{
	217	return a->ops->act(skb, a, res);
	218	}
	219
	220	static inline int tc_classify(struct sk_buff skb, const struct tcf_proto tp,
	221	struct tcf_result *res)
	222	{
	223	return tp->classify(skb, tp, res);
	224	}
	225
	226	static inline void tc_wrapper_init(void)
	227	{
	228	}
	229
	230	#endif
	231
	232	#endif /* __NET_TC_WRAPPER_H */