[linux-block.git] / net / netfilter / nf_tables_core.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008 Patrick McHardy <kaber@trash.net>
 *
 * Development of this code funded by Astaro AG (http://www.astaro.com/)
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/rculist.h>
#include <linux/skbuff.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/static_key.h>
#include <linux/netfilter/nfnetlink.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_tables.h>
#include <net/netfilter/nf_log.h>
#include <net/netfilter/nft_meta.h>

#if defined(CONFIG_RETPOLINE) && defined(CONFIG_X86)

static struct static_key_false nf_tables_skip_direct_calls;

static bool nf_skip_indirect_calls(void)
{
	return static_branch_likely(&nf_tables_skip_direct_calls);
}

static void __init nf_skip_indirect_calls_enable(void)
{
	if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE))
		static_branch_enable(&nf_tables_skip_direct_calls);
}
#else
static inline bool nf_skip_indirect_calls(void) { return false; }

static inline void nf_skip_indirect_calls_enable(void) { }
#endif

static noinline void __nft_trace_packet(const struct nft_pktinfo *pkt,
					const struct nft_verdict *verdict,
					const struct nft_rule_dp *rule,
					struct nft_traceinfo *info,
					enum nft_trace_types type)
{
	if (!info->trace || !info->nf_trace)
		return;

	info->type = type;

	nft_trace_notify(pkt, verdict, rule, info);
}

static inline void nft_trace_packet(const struct nft_pktinfo *pkt,
				    struct nft_verdict *verdict,
				    struct nft_traceinfo *info,
				    const struct nft_rule_dp *rule,
				    enum nft_trace_types type)
{
	if (static_branch_unlikely(&nft_trace_enabled)) {
		info->nf_trace = pkt->skb->nf_trace;
		__nft_trace_packet(pkt, verdict, rule, info, type);
	}
}

static inline void nft_trace_copy_nftrace(const struct nft_pktinfo *pkt,
					  struct nft_traceinfo *info)
{
	if (static_branch_unlikely(&nft_trace_enabled))
		info->nf_trace = pkt->skb->nf_trace;
}

static void nft_bitwise_fast_eval(const struct nft_expr *expr,
				  struct nft_regs *regs)
{
	const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
	u32 *src = &regs->data[priv->sreg];
	u32 *dst = &regs->data[priv->dreg];

	*dst = (*src & priv->mask) ^ priv->xor;
}

static void nft_cmp_fast_eval(const struct nft_expr *expr,
			      struct nft_regs *regs)
{
	const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);

	if (((regs->data[priv->sreg] & priv->mask) == priv->data) ^ priv->inv)
		return;
	regs->verdict.code = NFT_BREAK;
}

static void nft_cmp16_fast_eval(const struct nft_expr *expr,
				struct nft_regs *regs)
{
	const struct nft_cmp16_fast_expr *priv = nft_expr_priv(expr);
	const u64 *reg_data = (const u64 *)&regs->data[priv->sreg];
	const u64 *mask = (const u64 *)&priv->mask;
	const u64 *data = (const u64 *)&priv->data;

	if (((reg_data[0] & mask[0]) == data[0] &&
	    ((reg_data[1] & mask[1]) == data[1])) ^ priv->inv)
		return;
	regs->verdict.code = NFT_BREAK;
}

static noinline void __nft_trace_verdict(const struct nft_pktinfo *pkt,
					 struct nft_traceinfo *info,
					 const struct nft_rule_dp *rule,
					 const struct nft_regs *regs)
{
	enum nft_trace_types type;

	switch (regs->verdict.code) {
	case NFT_CONTINUE:
	case NFT_RETURN:
		type = NFT_TRACETYPE_RETURN;
		break;
	case NF_STOLEN:
		type = NFT_TRACETYPE_RULE;
		/* can't access skb->nf_trace; use copy */
		break;
	default:
		type = NFT_TRACETYPE_RULE;

		if (info->trace)
			info->nf_trace = pkt->skb->nf_trace;
		break;
	}

	__nft_trace_packet(pkt, &regs->verdict, rule, info, type);
}

static inline void nft_trace_verdict(const struct nft_pktinfo *pkt,
				     struct nft_traceinfo *info,
				     const struct nft_rule_dp *rule,
				     const struct nft_regs *regs)
{
	if (static_branch_unlikely(&nft_trace_enabled))
		__nft_trace_verdict(pkt, info, rule, regs);
}

static bool nft_payload_fast_eval(const struct nft_expr *expr,
				  struct nft_regs *regs,
				  const struct nft_pktinfo *pkt)
{
	const struct nft_payload *priv = nft_expr_priv(expr);
	const struct sk_buff *skb = pkt->skb;
	u32 *dest = &regs->data[priv->dreg];
	unsigned char *ptr;

	if (priv->base == NFT_PAYLOAD_NETWORK_HEADER)
		ptr = skb_network_header(skb);
	else {
		if (!(pkt->flags & NFT_PKTINFO_L4PROTO))
			return false;
		ptr = skb_network_header(skb) + nft_thoff(pkt);
	}

	ptr += priv->offset;

	if (unlikely(ptr + priv->len > skb_tail_pointer(skb)))
		return false;

	*dest = 0;
	if (priv->len == 2)
		*(u16 *)dest = *(u16 *)ptr;
	else if (priv->len == 4)
		*(u32 *)dest = *(u32 *)ptr;
	else
		*(u8 *)dest = *(u8 *)ptr;
	return true;
}

DEFINE_STATIC_KEY_FALSE(nft_counters_enabled);

static noinline void nft_update_chain_stats(const struct nft_chain *chain,
					    const struct nft_pktinfo *pkt)
{
	struct nft_base_chain *base_chain;
	struct nft_stats __percpu *pstats;
	struct nft_stats *stats;

	base_chain = nft_base_chain(chain);

	pstats = READ_ONCE(base_chain->stats);
	if (pstats) {
		local_bh_disable();
		stats = this_cpu_ptr(pstats);
		u64_stats_update_begin(&stats->syncp);
		stats->pkts++;
		stats->bytes += pkt->skb->len;
		u64_stats_update_end(&stats->syncp);
		local_bh_enable();
	}
}

struct nft_jumpstack {
	const struct nft_rule_dp *rule;
};

static void expr_call_ops_eval(const struct nft_expr *expr,
			       struct nft_regs *regs,
			       struct nft_pktinfo *pkt)
{
#ifdef CONFIG_RETPOLINE
	unsigned long e;

	if (nf_skip_indirect_calls())
		goto indirect_call;

	e = (unsigned long)expr->ops->eval;
#define X(e, fun) \
	do { if ((e) == (unsigned long)(fun)) \
		return fun(expr, regs, pkt); } while (0)

	X(e, nft_payload_eval);
	X(e, nft_cmp_eval);
	X(e, nft_counter_eval);
	X(e, nft_meta_get_eval);
	X(e, nft_lookup_eval);
#if IS_ENABLED(CONFIG_NFT_CT)
	X(e, nft_ct_get_fast_eval);
#endif
	X(e, nft_range_eval);
	X(e, nft_immediate_eval);
	X(e, nft_byteorder_eval);
	X(e, nft_dynset_eval);
	X(e, nft_rt_get_eval);
	X(e, nft_bitwise_eval);
	X(e, nft_objref_eval);
	X(e, nft_objref_map_eval);
#undef  X
indirect_call:
#endif /* CONFIG_RETPOLINE */
	expr->ops->eval(expr, regs, pkt);
}

#define nft_rule_expr_first(rule)	(struct nft_expr *)&rule->data[0]
#define nft_rule_expr_next(expr)	((void *)expr) + expr->ops->size
#define nft_rule_expr_last(rule)	(struct nft_expr *)&rule->data[rule->dlen]

#define nft_rule_dp_for_each_expr(expr, last, rule) \
        for ((expr) = nft_rule_expr_first(rule), (last) = nft_rule_expr_last(rule); \
             (expr) != (last); \
             (expr) = nft_rule_expr_next(expr))

unsigned int
nft_do_chain(struct nft_pktinfo *pkt, void *priv)
{
	const struct nft_chain *chain = priv, *basechain = chain;
	const struct net *net = nft_net(pkt);
	const struct nft_expr *expr, *last;
	const struct nft_rule_dp *rule;
	struct nft_regs regs = {};
	unsigned int stackptr = 0;
	struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE];
	bool genbit = READ_ONCE(net->nft.gencursor);
	struct nft_rule_blob *blob;
	struct nft_traceinfo info;

	info.trace = false;
	if (static_branch_unlikely(&nft_trace_enabled))
		nft_trace_init(&info, pkt, basechain);
do_chain:
	if (genbit)
		blob = rcu_dereference(chain->blob_gen_1);
	else
		blob = rcu_dereference(chain->blob_gen_0);

	rule = (struct nft_rule_dp *)blob->data;
next_rule:
	regs.verdict.code = NFT_CONTINUE;
	for (; !rule->is_last ; rule = nft_rule_next(rule)) {
		nft_rule_dp_for_each_expr(expr, last, rule) {
			if (expr->ops == &nft_cmp_fast_ops)
				nft_cmp_fast_eval(expr, &regs);
			else if (expr->ops == &nft_cmp16_fast_ops)
				nft_cmp16_fast_eval(expr, &regs);
			else if (expr->ops == &nft_bitwise_fast_ops)
				nft_bitwise_fast_eval(expr, &regs);
			else if (expr->ops != &nft_payload_fast_ops ||
				 !nft_payload_fast_eval(expr, &regs, pkt))
				expr_call_ops_eval(expr, &regs, pkt);

			if (regs.verdict.code != NFT_CONTINUE)
				break;
		}

		switch (regs.verdict.code) {
		case NFT_BREAK:
			regs.verdict.code = NFT_CONTINUE;
			nft_trace_copy_nftrace(pkt, &info);
			continue;
		case NFT_CONTINUE:
			nft_trace_packet(pkt, &regs.verdict,  &info, rule,
					 NFT_TRACETYPE_RULE);
			continue;
		}
		break;
	}

	nft_trace_verdict(pkt, &info, rule, &regs);

	switch (regs.verdict.code & NF_VERDICT_MASK) {
	case NF_ACCEPT:
	case NF_DROP:
	case NF_QUEUE:
	case NF_STOLEN:
		return regs.verdict.code;
	}

	switch (regs.verdict.code) {
	case NFT_JUMP:
		if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE))
			return NF_DROP;
		jumpstack[stackptr].rule = nft_rule_next(rule);
		stackptr++;
		fallthrough;
	case NFT_GOTO:
		chain = regs.verdict.chain;
		goto do_chain;
	case NFT_CONTINUE:
	case NFT_RETURN:
		break;
	default:
		WARN_ON_ONCE(1);
	}

	if (stackptr > 0) {
		stackptr--;
		rule = jumpstack[stackptr].rule;
		goto next_rule;
	}

	nft_trace_packet(pkt, &regs.verdict, &info, NULL, NFT_TRACETYPE_POLICY);

	if (static_branch_unlikely(&nft_counters_enabled))
		nft_update_chain_stats(basechain, pkt);

	return nft_base_chain(basechain)->policy;
}
EXPORT_SYMBOL_GPL(nft_do_chain);

static struct nft_expr_type *nft_basic_types[] = {
	&nft_imm_type,
	&nft_cmp_type,
	&nft_lookup_type,
	&nft_bitwise_type,
	&nft_byteorder_type,
	&nft_payload_type,
	&nft_dynset_type,
	&nft_range_type,
	&nft_meta_type,
	&nft_rt_type,
	&nft_exthdr_type,
	&nft_last_type,
	&nft_counter_type,
	&nft_objref_type,
	&nft_inner_type,
};

static struct nft_object_type *nft_basic_objects[] = {
#ifdef CONFIG_NETWORK_SECMARK
	&nft_secmark_obj_type,
#endif
	&nft_counter_obj_type,
};

int __init nf_tables_core_module_init(void)
{
	int err, i, j = 0;

	nft_counter_init_seqcount();

	for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) {
		err = nft_register_obj(nft_basic_objects[i]);
		if (err)
			goto err;
	}

	for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) {
		err = nft_register_expr(nft_basic_types[j]);
		if (err)
			goto err;
	}

	nf_skip_indirect_calls_enable();

	return 0;

err:
	while (j-- > 0)
		nft_unregister_expr(nft_basic_types[j]);

	while (i-- > 0)
		nft_unregister_obj(nft_basic_objects[i]);

	return err;
}

void nf_tables_core_module_exit(void)
{
	int i;

	i = ARRAY_SIZE(nft_basic_types);
	while (i-- > 0)
		nft_unregister_expr(nft_basic_types[i]);

	i = ARRAY_SIZE(nft_basic_objects);
	while (i-- > 0)
		nft_unregister_obj(nft_basic_objects[i]);
}
Commit	Line	Data
d2912cb1	1	// SPDX-License-Identifier: GPL-2.0-only
96518518 PM	2	/*
	3	* Copyright (c) 2008 Patrick McHardy <kaber@trash.net>
	4	*
96518518 PM	5	* Development of this code funded by Astaro AG (http://www.astaro.com/)
	6	*/
	7
a81b2ce8	8	#include <linux/kernel.h>
96518518 PM	9	#include <linux/module.h>
	10	#include <linux/init.h>
	11	#include <linux/list.h>
	12	#include <linux/rculist.h>
	13	#include <linux/skbuff.h>
	14	#include <linux/netlink.h>
	15	#include <linux/netfilter.h>
e639f7ab	16	#include <linux/static_key.h>
96518518 PM	17	#include <linux/netfilter/nfnetlink.h>
	18	#include <linux/netfilter/nf_tables.h>
	19	#include <net/netfilter/nf_tables_core.h>
	20	#include <net/netfilter/nf_tables.h>
b5bc89bf	21	#include <net/netfilter/nf_log.h>
30e103fe	22	#include <net/netfilter/nft_meta.h>
96518518	23
d8d76062 FW	24	#if defined(CONFIG_RETPOLINE) && defined(CONFIG_X86)
	25
	26	static struct static_key_false nf_tables_skip_direct_calls;
	27
	28	static bool nf_skip_indirect_calls(void)
	29	{
	30	return static_branch_likely(&nf_tables_skip_direct_calls);
	31	}
	32
	33	static void __init nf_skip_indirect_calls_enable(void)
	34	{
	35	if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE))
	36	static_branch_enable(&nf_tables_skip_direct_calls);
	37	}
	38	#else
	39	static inline bool nf_skip_indirect_calls(void) { return false; }
	40
	41	static inline void nf_skip_indirect_calls_enable(void) { }
	42	#endif
	43
698bb828	44	static noinline void __nft_trace_packet(const struct nft_pktinfo *pkt,
0a202145	45	const struct nft_verdict *verdict,
46df4175	46	const struct nft_rule_dp *rule,
698bb828	47	struct nft_traceinfo *info,
e65eebec	48	enum nft_trace_types type)
01ef16c2	49	{
e34b9ed9	50	if (!info->trace \|\| !info->nf_trace)
33d5a7b1 FW	51	return;
33d5a7b1 FW	52
33d5a7b1 FW	53	info->type = type;
33d5a7b1 FW	54
46df4175	55	nft_trace_notify(pkt, verdict, rule, info);
01ef16c2 PM	56	}
01ef16c2 PM	57
399a14ec	58	static inline void nft_trace_packet(const struct nft_pktinfo *pkt,
0a202145	59	struct nft_verdict *verdict,
399a14ec	60	struct nft_traceinfo *info,
2c865a8a	61	const struct nft_rule_dp *rule,
33d5a7b1	62	enum nft_trace_types type)
01ef16c2	63	{
e639f7ab	64	if (static_branch_unlikely(&nft_trace_enabled)) {
e34b9ed9	65	info->nf_trace = pkt->skb->nf_trace;
46df4175	66	__nft_trace_packet(pkt, verdict, rule, info, type);
33d5a7b1	67	}
01ef16c2 PM	68	}
01ef16c2 PM	69
399a14ec FW	70	static inline void nft_trace_copy_nftrace(const struct nft_pktinfo *pkt,
399a14ec FW	71	struct nft_traceinfo *info)
e34b9ed9	72	{
2a1d6abd FW	73	if (static_branch_unlikely(&nft_trace_enabled))
2a1d6abd FW	74	info->nf_trace = pkt->skb->nf_trace;
e34b9ed9 FW	75	}
e34b9ed9 FW	76
10fdd6d8 PS	77	static void nft_bitwise_fast_eval(const struct nft_expr *expr,
	78	struct nft_regs *regs)
	79	{
	80	const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
	81	u32 *src = &regs->data[priv->sreg];
	82	u32 *dst = &regs->data[priv->dreg];
	83
	84	dst = (src & priv->mask) ^ priv->xor;
	85	}
	86
cb7dbfd0	87	static void nft_cmp_fast_eval(const struct nft_expr *expr,
a55e22e9	88	struct nft_regs *regs)
cb7dbfd0 PM	89	{
cb7dbfd0 PM	90	const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
cb7dbfd0	91
5f48846d	92	if (((regs->data[priv->sreg] & priv->mask) == priv->data) ^ priv->inv)
cb7dbfd0	93	return;
a55e22e9	94	regs->verdict.code = NFT_BREAK;
cb7dbfd0 PM	95	}
cb7dbfd0 PM	96
23f68d46 PNA	97	static void nft_cmp16_fast_eval(const struct nft_expr *expr,
	98	struct nft_regs *regs)
	99	{
	100	const struct nft_cmp16_fast_expr *priv = nft_expr_priv(expr);
	101	const u64 reg_data = (const u64 )&regs->data[priv->sreg];
	102	const u64 mask = (const u64 )&priv->mask;
	103	const u64 data = (const u64 )&priv->data;
	104
	105	if (((reg_data[0] & mask[0]) == data[0] &&
	106	((reg_data[1] & mask[1]) == data[1])) ^ priv->inv)
	107	return;
	108	regs->verdict.code = NFT_BREAK;
	109	}
	110
698bb828 FW	111	static noinline void __nft_trace_verdict(const struct nft_pktinfo *pkt,
698bb828 FW	112	struct nft_traceinfo *info,
46df4175	113	const struct nft_rule_dp *rule,
4765473f PNA	114	const struct nft_regs *regs)
	115	{
	116	enum nft_trace_types type;
	117
	118	switch (regs->verdict.code) {
	119	case NFT_CONTINUE:
	120	case NFT_RETURN:
	121	type = NFT_TRACETYPE_RETURN;
	122	break;
e34b9ed9 FW	123	case NF_STOLEN:
	124	type = NFT_TRACETYPE_RULE;
	125	/* can't access skb->nf_trace; use copy */
	126	break;
4765473f PNA	127	default:
4765473f PNA	128	type = NFT_TRACETYPE_RULE;
399a14ec FW	129
399a14ec FW	130	if (info->trace)
698bb828	131	info->nf_trace = pkt->skb->nf_trace;
4765473f PNA	132	break;
	133	}
	134
46df4175	135	__nft_trace_packet(pkt, &regs->verdict, rule, info, type);
4765473f PNA	136	}
4765473f PNA	137
698bb828 FW	138	static inline void nft_trace_verdict(const struct nft_pktinfo *pkt,
698bb828 FW	139	struct nft_traceinfo *info,
2c865a8a	140	const struct nft_rule_dp *rule,
4765473f PNA	141	const struct nft_regs *regs)
4765473f PNA	142	{
46df4175 FW	143	if (static_branch_unlikely(&nft_trace_enabled))
46df4175 FW	144	__nft_trace_verdict(pkt, info, rule, regs);
4765473f PNA	145	}
4765473f PNA	146
c29b72e0	147	static bool nft_payload_fast_eval(const struct nft_expr *expr,
a55e22e9	148	struct nft_regs *regs,
c29b72e0 PM	149	const struct nft_pktinfo *pkt)
	150	{
	151	const struct nft_payload *priv = nft_expr_priv(expr);
	152	const struct sk_buff *skb = pkt->skb;
49499c3e	153	u32 *dest = &regs->data[priv->dreg];
c29b72e0 PM	154	unsigned char *ptr;
	155
	156	if (priv->base == NFT_PAYLOAD_NETWORK_HEADER)
	157	ptr = skb_network_header(skb);
a20877b5	158	else {
b5bdc6f9	159	if (!(pkt->flags & NFT_PKTINFO_L4PROTO))
a20877b5	160	return false;
2d7b4ace	161	ptr = skb_network_header(skb) + nft_thoff(pkt);
a20877b5	162	}
c29b72e0 PM	163
	164	ptr += priv->offset;
	165
8dc3c2b8	166	if (unlikely(ptr + priv->len > skb_tail_pointer(skb)))
c29b72e0 PM	167	return false;
c29b72e0 PM	168
49499c3e	169	*dest = 0;
c29b72e0	170	if (priv->len == 2)
fad136ea	171	(u16 )dest = (u16 )ptr;
c29b72e0	172	else if (priv->len == 4)
fad136ea	173	(u32 )dest = (u32 )ptr;
c29b72e0	174	else
fad136ea	175	(u8 )dest = (u8 )ptr;
c29b72e0 PM	176	return true;
	177	}
	178
9f08ea84 PNA	179	DEFINE_STATIC_KEY_FALSE(nft_counters_enabled);
	180
	181	static noinline void nft_update_chain_stats(const struct nft_chain *chain,
	182	const struct nft_pktinfo *pkt)
	183	{
00924094	184	struct nft_base_chain *base_chain;
a9f5e78c	185	struct nft_stats __percpu *pstats;
9f08ea84 PNA	186	struct nft_stats *stats;
9f08ea84 PNA	187
00924094	188	base_chain = nft_base_chain(chain);
00924094	189
a9f5e78c LR	190	pstats = READ_ONCE(base_chain->stats);
	191	if (pstats) {
	192	local_bh_disable();
	193	stats = this_cpu_ptr(pstats);
00924094 FW	194	u64_stats_update_begin(&stats->syncp);
	195	stats->pkts++;
	196	stats->bytes += pkt->skb->len;
	197	u64_stats_update_end(&stats->syncp);
a9f5e78c	198	local_bh_enable();
00924094	199	}
9f08ea84 PNA	200	}
9f08ea84 PNA	201
0ca743a5	202	struct nft_jumpstack {
2c865a8a	203	const struct nft_rule_dp *rule;
0ca743a5 PNA	204	};
0ca743a5 PNA	205
222440b4 FW	206	static void expr_call_ops_eval(const struct nft_expr *expr,
	207	struct nft_regs *regs,
	208	struct nft_pktinfo *pkt)
	209	{
10870dd8	210	#ifdef CONFIG_RETPOLINE
d8d76062 FW	211	unsigned long e;
	212
	213	if (nf_skip_indirect_calls())
	214	goto indirect_call;
	215
	216	e = (unsigned long)expr->ops->eval;
10870dd8 FW	217	#define X(e, fun) \
	218	do { if ((e) == (unsigned long)(fun)) \
	219	return fun(expr, regs, pkt); } while (0)
	220
	221	X(e, nft_payload_eval);
	222	X(e, nft_cmp_eval);
023223df	223	X(e, nft_counter_eval);
10870dd8 FW	224	X(e, nft_meta_get_eval);
10870dd8 FW	225	X(e, nft_lookup_eval);
d9e78914 FW	226	#if IS_ENABLED(CONFIG_NFT_CT)
	227	X(e, nft_ct_get_fast_eval);
	228	#endif
10870dd8 FW	229	X(e, nft_range_eval);
	230	X(e, nft_immediate_eval);
	231	X(e, nft_byteorder_eval);
	232	X(e, nft_dynset_eval);
	233	X(e, nft_rt_get_eval);
	234	X(e, nft_bitwise_eval);
2032e907 FW	235	X(e, nft_objref_eval);
2032e907 FW	236	X(e, nft_objref_map_eval);
10870dd8	237	#undef X
d8d76062	238	indirect_call:
10870dd8 FW	239	#endif /* CONFIG_RETPOLINE */
10870dd8 FW	240	expr->ops->eval(expr, regs, pkt);
222440b4 FW	241	}
222440b4 FW	242
2c865a8a PNA	243	#define nft_rule_expr_first(rule) (struct nft_expr *)&rule->data[0]
	244	#define nft_rule_expr_next(expr) ((void *)expr) + expr->ops->size
	245	#define nft_rule_expr_last(rule) (struct nft_expr *)&rule->data[rule->dlen]
2c865a8a PNA	246
	247	#define nft_rule_dp_for_each_expr(expr, last, rule) \
	248	for ((expr) = nft_rule_expr_first(rule), (last) = nft_rule_expr_last(rule); \
	249	(expr) != (last); \
	250	(expr) = nft_rule_expr_next(expr))
	251
0ca743a5	252	unsigned int
06198b34	253	nft_do_chain(struct nft_pktinfo pkt, void priv)
96518518	254	{
06198b34	255	const struct nft_chain chain = priv, basechain = chain;
0e5a1c7e	256	const struct net *net = nft_net(pkt);
96518518	257	const struct nft_expr expr, last;
d4d89e65	258	const struct nft_rule_dp *rule;
4c905f67	259	struct nft_regs regs = {};
96518518	260	unsigned int stackptr = 0;
0ca743a5	261	struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE];
0cbc06b3	262	bool genbit = READ_ONCE(net->nft.gencursor);
2c865a8a	263	struct nft_rule_blob *blob;
33d5a7b1	264	struct nft_traceinfo info;
96518518	265
e639f7ab FW	266	info.trace = false;
e639f7ab FW	267	if (static_branch_unlikely(&nft_trace_enabled))
0a202145	268	nft_trace_init(&info, pkt, basechain);
96518518	269	do_chain:
0cbc06b3	270	if (genbit)
2c865a8a	271	blob = rcu_dereference(chain->blob_gen_1);
0cbc06b3	272	else
2c865a8a	273	blob = rcu_dereference(chain->blob_gen_0);
0cbc06b3	274
2c865a8a	275	rule = (struct nft_rule_dp *)blob->data;
96518518	276	next_rule:
a55e22e9	277	regs.verdict.code = NFT_CONTINUE;
d4d89e65	278	for (; !rule->is_last ; rule = nft_rule_next(rule)) {
2c865a8a	279	nft_rule_dp_for_each_expr(expr, last, rule) {
cb7dbfd0	280	if (expr->ops == &nft_cmp_fast_ops)
a55e22e9	281	nft_cmp_fast_eval(expr, &regs);
23f68d46 PNA	282	else if (expr->ops == &nft_cmp16_fast_ops)
23f68d46 PNA	283	nft_cmp16_fast_eval(expr, &regs);
10fdd6d8 PS	284	else if (expr->ops == &nft_bitwise_fast_ops)
10fdd6d8 PS	285	nft_bitwise_fast_eval(expr, &regs);
c29b72e0	286	else if (expr->ops != &nft_payload_fast_ops \|\|
a55e22e9	287	!nft_payload_fast_eval(expr, &regs, pkt))
222440b4	288	expr_call_ops_eval(expr, &regs, pkt);
cb7dbfd0	289
a55e22e9	290	if (regs.verdict.code != NFT_CONTINUE)
96518518 PM	291	break;
	292	}
	293
a55e22e9	294	switch (regs.verdict.code) {
96518518	295	case NFT_BREAK:
a55e22e9	296	regs.verdict.code = NFT_CONTINUE;
399a14ec	297	nft_trace_copy_nftrace(pkt, &info);
3b084e99	298	continue;
96518518	299	case NFT_CONTINUE:
0a202145	300	nft_trace_packet(pkt, &regs.verdict, &info, rule,
e65eebec	301	NFT_TRACETYPE_RULE);
96518518 PM	302	continue;
	303	}
	304	break;
	305	}
	306
698bb828	307	nft_trace_verdict(pkt, &info, rule, &regs);
4765473f	308
a55e22e9	309	switch (regs.verdict.code & NF_VERDICT_MASK) {
96518518 PM	310	case NF_ACCEPT:
	311	case NF_DROP:
	312	case NF_QUEUE:
5efa0fc6	313	case NF_STOLEN:
a55e22e9	314	return regs.verdict.code;
e569bdab EL	315	}
e569bdab EL	316
a55e22e9	317	switch (regs.verdict.code) {
96518518	318	case NFT_JUMP:
adc972c5 TY	319	if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE))
adc972c5 TY	320	return NF_DROP;
2c865a8a	321	jumpstack[stackptr].rule = nft_rule_next(rule);
96518518	322	stackptr++;
954d8297	323	fallthrough;
96518518	324	case NFT_GOTO:
a55e22e9	325	chain = regs.verdict.chain;
96518518	326	goto do_chain;
354bf5a0	327	case NFT_CONTINUE:
96518518	328	case NFT_RETURN:
7e9bc10d	329	break;
96518518	330	default:
690d5417	331	WARN_ON_ONCE(1);
96518518 PM	332	}
	333
	334	if (stackptr > 0) {
	335	stackptr--;
2c865a8a	336	rule = jumpstack[stackptr].rule;
96518518 PM	337	goto next_rule;
	338	}
	339
0a202145	340	nft_trace_packet(pkt, &regs.verdict, &info, NULL, NFT_TRACETYPE_POLICY);
5467a512	341
9f08ea84 PNA	342	if (static_branch_unlikely(&nft_counters_enabled))
9f08ea84 PNA	343	nft_update_chain_stats(basechain, pkt);
b5bc89bf	344
5467a512	345	return nft_base_chain(basechain)->policy;
96518518	346	}
3876d22d	347	EXPORT_SYMBOL_GPL(nft_do_chain);
96518518	348
4e24877e LZ	349	static struct nft_expr_type *nft_basic_types[] = {
	350	&nft_imm_type,
	351	&nft_cmp_type,
	352	&nft_lookup_type,
	353	&nft_bitwise_type,
	354	&nft_byteorder_type,
	355	&nft_payload_type,
	356	&nft_dynset_type,
	357	&nft_range_type,
8a22543c	358	&nft_meta_type,
ae1bc6a9	359	&nft_rt_type,
d0103158	360	&nft_exthdr_type,
836382dc	361	&nft_last_type,
023223df	362	&nft_counter_type,
d037abc2	363	&nft_objref_type,
3a07327d	364	&nft_inner_type,
4e24877e LZ	365	};
4e24877e LZ	366
fb961945 CG	367	static struct nft_object_type *nft_basic_objects[] = {
	368	#ifdef CONFIG_NETWORK_SECMARK
	369	&nft_secmark_obj_type,
	370	#endif
023223df	371	&nft_counter_obj_type,
fb961945 CG	372	};
fb961945 CG	373
96518518 PM	374	int __init nf_tables_core_module_init(void)
96518518 PM	375	{
fb961945 CG	376	int err, i, j = 0;
fb961945 CG	377
023223df PNA	378	nft_counter_init_seqcount();
023223df PNA	379
fb961945 CG	380	for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) {
	381	err = nft_register_obj(nft_basic_objects[i]);
	382	if (err)
	383	goto err;
	384	}
96518518	385
fb961945 CG	386	for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) {
fb961945 CG	387	err = nft_register_expr(nft_basic_types[j]);
4e24877e LZ	388	if (err)
	389	goto err;
	390	}
96518518	391
d8d76062 FW	392	nf_skip_indirect_calls_enable();
d8d76062 FW	393
0f3cd9b3	394	return 0;
4e24877e LZ	395
4e24877e LZ	396	err:
fb961945 CG	397	while (j-- > 0)
	398	nft_unregister_expr(nft_basic_types[j]);
	399
4e24877e	400	while (i-- > 0)
fb961945 CG	401	nft_unregister_obj(nft_basic_objects[i]);
fb961945 CG	402
96518518 PM	403	return err;
	404	}
	405
	406	void nf_tables_core_module_exit(void)
	407	{
4e24877e LZ	408	int i;
	409
	410	i = ARRAY_SIZE(nft_basic_types);
	411	while (i-- > 0)
	412	nft_unregister_expr(nft_basic_types[i]);
fb961945 CG	413
	414	i = ARRAY_SIZE(nft_basic_objects);
	415	while (i-- > 0)
	416	nft_unregister_obj(nft_basic_objects[i]);
96518518	417	}