[linux-block.git] / net / netfilter / nft_bitwise.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
 *
 * Development of this code funded by Astaro AG (http://www.astaro.com/)
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_offload.h>

struct nft_bitwise {
	u8			sreg;
	u8			dreg;
	enum nft_bitwise_ops	op:8;
	u8			len;
	struct nft_data		mask;
	struct nft_data		xor;
	struct nft_data		data;
};

static void nft_bitwise_eval_bool(u32 *dst, const u32 *src,
				  const struct nft_bitwise *priv)
{
	unsigned int i;

	for (i = 0; i < DIV_ROUND_UP(priv->len, 4); i++)
		dst[i] = (src[i] & priv->mask.data[i]) ^ priv->xor.data[i];
}

static void nft_bitwise_eval_lshift(u32 *dst, const u32 *src,
				    const struct nft_bitwise *priv)
{
	u32 shift = priv->data.data[0];
	unsigned int i;
	u32 carry = 0;

	for (i = DIV_ROUND_UP(priv->len, sizeof(u32)); i > 0; i--) {
		dst[i - 1] = (src[i - 1] << shift) | carry;
		carry = src[i - 1] >> (BITS_PER_TYPE(u32) - shift);
	}
}

static void nft_bitwise_eval_rshift(u32 *dst, const u32 *src,
				    const struct nft_bitwise *priv)
{
	u32 shift = priv->data.data[0];
	unsigned int i;
	u32 carry = 0;

	for (i = 0; i < DIV_ROUND_UP(priv->len, sizeof(u32)); i++) {
		dst[i] = carry | (src[i] >> shift);
		carry = src[i] << (BITS_PER_TYPE(u32) - shift);
	}
}

void nft_bitwise_eval(const struct nft_expr *expr,
		      struct nft_regs *regs, const struct nft_pktinfo *pkt)
{
	const struct nft_bitwise *priv = nft_expr_priv(expr);
	const u32 *src = &regs->data[priv->sreg];
	u32 *dst = &regs->data[priv->dreg];

	switch (priv->op) {
	case NFT_BITWISE_BOOL:
		nft_bitwise_eval_bool(dst, src, priv);
		break;
	case NFT_BITWISE_LSHIFT:
		nft_bitwise_eval_lshift(dst, src, priv);
		break;
	case NFT_BITWISE_RSHIFT:
		nft_bitwise_eval_rshift(dst, src, priv);
		break;
	}
}

static const struct nla_policy nft_bitwise_policy[NFTA_BITWISE_MAX + 1] = {
	[NFTA_BITWISE_SREG]	= { .type = NLA_U32 },
	[NFTA_BITWISE_DREG]	= { .type = NLA_U32 },
	[NFTA_BITWISE_LEN]	= { .type = NLA_U32 },
	[NFTA_BITWISE_MASK]	= { .type = NLA_NESTED },
	[NFTA_BITWISE_XOR]	= { .type = NLA_NESTED },
	[NFTA_BITWISE_OP]	= { .type = NLA_U32 },
	[NFTA_BITWISE_DATA]	= { .type = NLA_NESTED },
};

static int nft_bitwise_init_bool(struct nft_bitwise *priv,
				 const struct nlattr *const tb[])
{
	struct nft_data_desc mask, xor;
	int err;

	if (tb[NFTA_BITWISE_DATA])
		return -EINVAL;

	if (!tb[NFTA_BITWISE_MASK] ||
	    !tb[NFTA_BITWISE_XOR])
		return -EINVAL;

	err = nft_data_init(NULL, &priv->mask, sizeof(priv->mask), &mask,
			    tb[NFTA_BITWISE_MASK]);
	if (err < 0)
		return err;
	if (mask.type != NFT_DATA_VALUE || mask.len != priv->len) {
		err = -EINVAL;
		goto err1;
	}

	err = nft_data_init(NULL, &priv->xor, sizeof(priv->xor), &xor,
			    tb[NFTA_BITWISE_XOR]);
	if (err < 0)
		goto err1;
	if (xor.type != NFT_DATA_VALUE || xor.len != priv->len) {
		err = -EINVAL;
		goto err2;
	}

	return 0;
err2:
	nft_data_release(&priv->xor, xor.type);
err1:
	nft_data_release(&priv->mask, mask.type);
	return err;
}

static int nft_bitwise_init_shift(struct nft_bitwise *priv,
				  const struct nlattr *const tb[])
{
	struct nft_data_desc d;
	int err;

	if (tb[NFTA_BITWISE_MASK] ||
	    tb[NFTA_BITWISE_XOR])
		return -EINVAL;

	if (!tb[NFTA_BITWISE_DATA])
		return -EINVAL;

	err = nft_data_init(NULL, &priv->data, sizeof(priv->data), &d,
			    tb[NFTA_BITWISE_DATA]);
	if (err < 0)
		return err;
	if (d.type != NFT_DATA_VALUE || d.len != sizeof(u32) ||
	    priv->data.data[0] >= BITS_PER_TYPE(u32)) {
		nft_data_release(&priv->data, d.type);
		return -EINVAL;
	}

	return 0;
}

static int nft_bitwise_init(const struct nft_ctx *ctx,
			    const struct nft_expr *expr,
			    const struct nlattr * const tb[])
{
	struct nft_bitwise *priv = nft_expr_priv(expr);
	u32 len;
	int err;

	err = nft_parse_u32_check(tb[NFTA_BITWISE_LEN], U8_MAX, &len);
	if (err < 0)
		return err;

	priv->len = len;

	err = nft_parse_register_load(tb[NFTA_BITWISE_SREG], &priv->sreg,
				      priv->len);
	if (err < 0)
		return err;

	err = nft_parse_register_store(ctx, tb[NFTA_BITWISE_DREG],
				       &priv->dreg, NULL, NFT_DATA_VALUE,
				       priv->len);
	if (err < 0)
		return err;

	if (tb[NFTA_BITWISE_OP]) {
		priv->op = ntohl(nla_get_be32(tb[NFTA_BITWISE_OP]));
		switch (priv->op) {
		case NFT_BITWISE_BOOL:
		case NFT_BITWISE_LSHIFT:
		case NFT_BITWISE_RSHIFT:
			break;
		default:
			return -EOPNOTSUPP;
		}
	} else {
		priv->op = NFT_BITWISE_BOOL;
	}

	switch(priv->op) {
	case NFT_BITWISE_BOOL:
		err = nft_bitwise_init_bool(priv, tb);
		break;
	case NFT_BITWISE_LSHIFT:
	case NFT_BITWISE_RSHIFT:
		err = nft_bitwise_init_shift(priv, tb);
		break;
	}

	return err;
}

static int nft_bitwise_dump_bool(struct sk_buff *skb,
				 const struct nft_bitwise *priv)
{
	if (nft_data_dump(skb, NFTA_BITWISE_MASK, &priv->mask,
			  NFT_DATA_VALUE, priv->len) < 0)
		return -1;

	if (nft_data_dump(skb, NFTA_BITWISE_XOR, &priv->xor,
			  NFT_DATA_VALUE, priv->len) < 0)
		return -1;

	return 0;
}

static int nft_bitwise_dump_shift(struct sk_buff *skb,
				  const struct nft_bitwise *priv)
{
	if (nft_data_dump(skb, NFTA_BITWISE_DATA, &priv->data,
			  NFT_DATA_VALUE, sizeof(u32)) < 0)
		return -1;
	return 0;
}

static int nft_bitwise_dump(struct sk_buff *skb, const struct nft_expr *expr)
{
	const struct nft_bitwise *priv = nft_expr_priv(expr);
	int err = 0;

	if (nft_dump_register(skb, NFTA_BITWISE_SREG, priv->sreg))
		return -1;
	if (nft_dump_register(skb, NFTA_BITWISE_DREG, priv->dreg))
		return -1;
	if (nla_put_be32(skb, NFTA_BITWISE_LEN, htonl(priv->len)))
		return -1;
	if (nla_put_be32(skb, NFTA_BITWISE_OP, htonl(priv->op)))
		return -1;

	switch (priv->op) {
	case NFT_BITWISE_BOOL:
		err = nft_bitwise_dump_bool(skb, priv);
		break;
	case NFT_BITWISE_LSHIFT:
	case NFT_BITWISE_RSHIFT:
		err = nft_bitwise_dump_shift(skb, priv);
		break;
	}

	return err;
}

static struct nft_data zero;

static int nft_bitwise_offload(struct nft_offload_ctx *ctx,
			       struct nft_flow_rule *flow,
			       const struct nft_expr *expr)
{
	const struct nft_bitwise *priv = nft_expr_priv(expr);
	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

	if (priv->op != NFT_BITWISE_BOOL)
		return -EOPNOTSUPP;

	if (memcmp(&priv->xor, &zero, sizeof(priv->xor)) ||
	    priv->sreg != priv->dreg || priv->len != reg->len)
		return -EOPNOTSUPP;

	memcpy(&reg->mask, &priv->mask, sizeof(priv->mask));

	return 0;
}

static bool nft_bitwise_reduce(struct nft_regs_track *track,
			       const struct nft_expr *expr)
{
	const struct nft_bitwise *priv = nft_expr_priv(expr);
	const struct nft_bitwise *bitwise;
	unsigned int regcount;
	u8 dreg;
	int i;

	if (!track->regs[priv->sreg].selector)
		return false;

	bitwise = nft_expr_priv(expr);
	if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector &&
	    track->regs[priv->sreg].num_reg == 0 &&
	    track->regs[priv->dreg].bitwise &&
	    track->regs[priv->dreg].bitwise->ops == expr->ops &&
	    priv->sreg == bitwise->sreg &&
	    priv->dreg == bitwise->dreg &&
	    priv->op == bitwise->op &&
	    priv->len == bitwise->len &&
	    !memcmp(&priv->mask, &bitwise->mask, sizeof(priv->mask)) &&
	    !memcmp(&priv->xor, &bitwise->xor, sizeof(priv->xor)) &&
	    !memcmp(&priv->data, &bitwise->data, sizeof(priv->data))) {
		track->cur = expr;
		return true;
	}

	if (track->regs[priv->sreg].bitwise ||
	    track->regs[priv->sreg].num_reg != 0) {
		nft_reg_track_cancel(track, priv->dreg, priv->len);
		return false;
	}

	if (priv->sreg != priv->dreg) {
		nft_reg_track_update(track, track->regs[priv->sreg].selector,
				     priv->dreg, priv->len);
	}

	dreg = priv->dreg;
	regcount = DIV_ROUND_UP(priv->len, NFT_REG32_SIZE);
	for (i = 0; i < regcount; i++, dreg++)
		track->regs[priv->dreg].bitwise = expr;

	return false;
}

static const struct nft_expr_ops nft_bitwise_ops = {
	.type		= &nft_bitwise_type,
	.size		= NFT_EXPR_SIZE(sizeof(struct nft_bitwise)),
	.eval		= nft_bitwise_eval,
	.init		= nft_bitwise_init,
	.dump		= nft_bitwise_dump,
	.reduce		= nft_bitwise_reduce,
	.offload	= nft_bitwise_offload,
};

static int
nft_bitwise_extract_u32_data(const struct nlattr * const tb, u32 *out)
{
	struct nft_data_desc desc;
	struct nft_data data;
	int err = 0;

	err = nft_data_init(NULL, &data, sizeof(data), &desc, tb);
	if (err < 0)
		return err;

	if (desc.type != NFT_DATA_VALUE || desc.len != sizeof(u32)) {
		err = -EINVAL;
		goto err;
	}
	*out = data.data[0];
err:
	nft_data_release(&data, desc.type);
	return err;
}

static int nft_bitwise_fast_init(const struct nft_ctx *ctx,
				 const struct nft_expr *expr,
				 const struct nlattr * const tb[])
{
	struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
	int err;

	err = nft_parse_register_load(tb[NFTA_BITWISE_SREG], &priv->sreg,
				      sizeof(u32));
	if (err < 0)
		return err;

	err = nft_parse_register_store(ctx, tb[NFTA_BITWISE_DREG], &priv->dreg,
				       NULL, NFT_DATA_VALUE, sizeof(u32));
	if (err < 0)
		return err;

	if (tb[NFTA_BITWISE_DATA])
		return -EINVAL;

	if (!tb[NFTA_BITWISE_MASK] ||
	    !tb[NFTA_BITWISE_XOR])
		return -EINVAL;

	err = nft_bitwise_extract_u32_data(tb[NFTA_BITWISE_MASK], &priv->mask);
	if (err < 0)
		return err;

	err = nft_bitwise_extract_u32_data(tb[NFTA_BITWISE_XOR], &priv->xor);
	if (err < 0)
		return err;

	return 0;
}

static int
nft_bitwise_fast_dump(struct sk_buff *skb, const struct nft_expr *expr)
{
	const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
	struct nft_data data;

	if (nft_dump_register(skb, NFTA_BITWISE_SREG, priv->sreg))
		return -1;
	if (nft_dump_register(skb, NFTA_BITWISE_DREG, priv->dreg))
		return -1;
	if (nla_put_be32(skb, NFTA_BITWISE_LEN, htonl(sizeof(u32))))
		return -1;
	if (nla_put_be32(skb, NFTA_BITWISE_OP, htonl(NFT_BITWISE_BOOL)))
		return -1;

	data.data[0] = priv->mask;
	if (nft_data_dump(skb, NFTA_BITWISE_MASK, &data,
			  NFT_DATA_VALUE, sizeof(u32)) < 0)
		return -1;

	data.data[0] = priv->xor;
	if (nft_data_dump(skb, NFTA_BITWISE_XOR, &data,
			  NFT_DATA_VALUE, sizeof(u32)) < 0)
		return -1;

	return 0;
}

static int nft_bitwise_fast_offload(struct nft_offload_ctx *ctx,
				    struct nft_flow_rule *flow,
				    const struct nft_expr *expr)
{
	const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

	if (priv->xor || priv->sreg != priv->dreg || reg->len != sizeof(u32))
		return -EOPNOTSUPP;

	reg->mask.data[0] = priv->mask;
	return 0;
}

static bool nft_bitwise_fast_reduce(struct nft_regs_track *track,
				    const struct nft_expr *expr)
{
	const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
	const struct nft_bitwise_fast_expr *bitwise;

	if (!track->regs[priv->sreg].selector)
		return false;

	bitwise = nft_expr_priv(expr);
	if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector &&
	    track->regs[priv->dreg].bitwise &&
	    track->regs[priv->dreg].bitwise->ops == expr->ops &&
	    priv->sreg == bitwise->sreg &&
	    priv->dreg == bitwise->dreg &&
	    priv->mask == bitwise->mask &&
	    priv->xor == bitwise->xor) {
		track->cur = expr;
		return true;
	}

	if (track->regs[priv->sreg].bitwise) {
		nft_reg_track_cancel(track, priv->dreg, NFT_REG32_SIZE);
		return false;
	}

	if (priv->sreg != priv->dreg) {
		track->regs[priv->dreg].selector =
			track->regs[priv->sreg].selector;
	}
	track->regs[priv->dreg].bitwise = expr;

	return false;
}

const struct nft_expr_ops nft_bitwise_fast_ops = {
	.type		= &nft_bitwise_type,
	.size		= NFT_EXPR_SIZE(sizeof(struct nft_bitwise_fast_expr)),
	.eval		= NULL, /* inlined */
	.init		= nft_bitwise_fast_init,
	.dump		= nft_bitwise_fast_dump,
	.reduce		= nft_bitwise_fast_reduce,
	.offload	= nft_bitwise_fast_offload,
};

static const struct nft_expr_ops *
nft_bitwise_select_ops(const struct nft_ctx *ctx,
		       const struct nlattr * const tb[])
{
	int err;
	u32 len;

	if (!tb[NFTA_BITWISE_LEN] ||
	    !tb[NFTA_BITWISE_SREG] ||
	    !tb[NFTA_BITWISE_DREG])
		return ERR_PTR(-EINVAL);

	err = nft_parse_u32_check(tb[NFTA_BITWISE_LEN], U8_MAX, &len);
	if (err < 0)
		return ERR_PTR(err);

	if (len != sizeof(u32))
		return &nft_bitwise_ops;

	if (tb[NFTA_BITWISE_OP] &&
	    ntohl(nla_get_be32(tb[NFTA_BITWISE_OP])) != NFT_BITWISE_BOOL)
		return &nft_bitwise_ops;

	return &nft_bitwise_fast_ops;
}

struct nft_expr_type nft_bitwise_type __read_mostly = {
	.name		= "bitwise",
	.select_ops	= nft_bitwise_select_ops,
	.policy		= nft_bitwise_policy,
	.maxattr	= NFTA_BITWISE_MAX,
	.owner		= THIS_MODULE,
};

bool nft_expr_reduce_bitwise(struct nft_regs_track *track,
			     const struct nft_expr *expr)
{
	const struct nft_expr *last = track->last;
	const struct nft_expr *next;

	if (expr == last)
		return false;

	next = nft_expr_next(expr);
	if (next->ops == &nft_bitwise_ops)
		return nft_bitwise_reduce(track, next);
	else if (next->ops == &nft_bitwise_fast_ops)
		return nft_bitwise_fast_reduce(track, next);

	return false;
}
Commit	Line	Data
d2912cb1	1	// SPDX-License-Identifier: GPL-2.0-only
96518518	2	/*
ef1f7df9	3	* Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
96518518	4	*
96518518 PM	5	* Development of this code funded by Astaro AG (http://www.astaro.com/)
	6	*/
	7
	8	#include <linux/kernel.h>
	9	#include <linux/init.h>
	10	#include <linux/module.h>
	11	#include <linux/netlink.h>
	12	#include <linux/netfilter.h>
	13	#include <linux/netfilter/nf_tables.h>
	14	#include <net/netfilter/nf_tables_core.h>
	15	#include <net/netfilter/nf_tables.h>
bd8699e9	16	#include <net/netfilter/nf_tables_offload.h>
96518518 PM	17
96518518 PM	18	struct nft_bitwise {
4f16d25c	19	u8 sreg;
345023b0	20	u8 dreg;
9d1f9799	21	enum nft_bitwise_ops op:8;
96518518 PM	22	u8 len;
	23	struct nft_data mask;
	24	struct nft_data xor;
779f725e	25	struct nft_data data;
96518518 PM	26	};
96518518 PM	27
71d6ded3 JS	28	static void nft_bitwise_eval_bool(u32 dst, const u32 src,
	29	const struct nft_bitwise *priv)
	30	{
	31	unsigned int i;
	32
	33	for (i = 0; i < DIV_ROUND_UP(priv->len, 4); i++)
	34	dst[i] = (src[i] & priv->mask.data[i]) ^ priv->xor.data[i];
	35	}
	36
567d746b JS	37	static void nft_bitwise_eval_lshift(u32 dst, const u32 src,
	38	const struct nft_bitwise *priv)
	39	{
	40	u32 shift = priv->data.data[0];
	41	unsigned int i;
	42	u32 carry = 0;
	43
	44	for (i = DIV_ROUND_UP(priv->len, sizeof(u32)); i > 0; i--) {
	45	dst[i - 1] = (src[i - 1] << shift) \| carry;
	46	carry = src[i - 1] >> (BITS_PER_TYPE(u32) - shift);
	47	}
	48	}
	49
	50	static void nft_bitwise_eval_rshift(u32 dst, const u32 src,
	51	const struct nft_bitwise *priv)
	52	{
	53	u32 shift = priv->data.data[0];
	54	unsigned int i;
	55	u32 carry = 0;
	56
	57	for (i = 0; i < DIV_ROUND_UP(priv->len, sizeof(u32)); i++) {
	58	dst[i] = carry \| (src[i] >> shift);
	59	carry = src[i] << (BITS_PER_TYPE(u32) - shift);
	60	}
	61	}
	62
10870dd8 FW	63	void nft_bitwise_eval(const struct nft_expr *expr,
10870dd8 FW	64	struct nft_regs regs, const struct nft_pktinfo pkt)
96518518 PM	65	{
96518518 PM	66	const struct nft_bitwise *priv = nft_expr_priv(expr);
49499c3e PM	67	const u32 *src = &regs->data[priv->sreg];
49499c3e PM	68	u32 *dst = &regs->data[priv->dreg];
96518518	69
71d6ded3 JS	70	switch (priv->op) {
	71	case NFT_BITWISE_BOOL:
	72	nft_bitwise_eval_bool(dst, src, priv);
	73	break;
567d746b JS	74	case NFT_BITWISE_LSHIFT:
	75	nft_bitwise_eval_lshift(dst, src, priv);
	76	break;
	77	case NFT_BITWISE_RSHIFT:
	78	nft_bitwise_eval_rshift(dst, src, priv);
	79	break;
71d6ded3	80	}
96518518 PM	81	}
	82
	83	static const struct nla_policy nft_bitwise_policy[NFTA_BITWISE_MAX + 1] = {
	84	[NFTA_BITWISE_SREG] = { .type = NLA_U32 },
	85	[NFTA_BITWISE_DREG] = { .type = NLA_U32 },
	86	[NFTA_BITWISE_LEN] = { .type = NLA_U32 },
	87	[NFTA_BITWISE_MASK] = { .type = NLA_NESTED },
	88	[NFTA_BITWISE_XOR] = { .type = NLA_NESTED },
9d1f9799	89	[NFTA_BITWISE_OP] = { .type = NLA_U32 },
779f725e	90	[NFTA_BITWISE_DATA] = { .type = NLA_NESTED },
96518518 PM	91	};
96518518 PM	92
3f8d9eb0 JS	93	static int nft_bitwise_init_bool(struct nft_bitwise *priv,
	94	const struct nlattr *const tb[])
	95	{
049dee95	96	struct nft_data_desc mask, xor;
3f8d9eb0 JS	97	int err;
3f8d9eb0 JS	98
779f725e JS	99	if (tb[NFTA_BITWISE_DATA])
	100	return -EINVAL;
	101
3f8d9eb0 JS	102	if (!tb[NFTA_BITWISE_MASK] \|\|
	103	!tb[NFTA_BITWISE_XOR])
	104	return -EINVAL;
	105
049dee95	106	err = nft_data_init(NULL, &priv->mask, sizeof(priv->mask), &mask,
3f8d9eb0 JS	107	tb[NFTA_BITWISE_MASK]);
	108	if (err < 0)
	109	return err;
049dee95	110	if (mask.type != NFT_DATA_VALUE \|\| mask.len != priv->len) {
3f8d9eb0 JS	111	err = -EINVAL;
	112	goto err1;
	113	}
	114
049dee95	115	err = nft_data_init(NULL, &priv->xor, sizeof(priv->xor), &xor,
3f8d9eb0 JS	116	tb[NFTA_BITWISE_XOR]);
	117	if (err < 0)
	118	goto err1;
049dee95	119	if (xor.type != NFT_DATA_VALUE \|\| xor.len != priv->len) {
3f8d9eb0 JS	120	err = -EINVAL;
	121	goto err2;
	122	}
	123
	124	return 0;
	125	err2:
049dee95	126	nft_data_release(&priv->xor, xor.type);
3f8d9eb0	127	err1:
049dee95	128	nft_data_release(&priv->mask, mask.type);
3f8d9eb0 JS	129	return err;
	130	}
	131
567d746b JS	132	static int nft_bitwise_init_shift(struct nft_bitwise *priv,
	133	const struct nlattr *const tb[])
	134	{
	135	struct nft_data_desc d;
	136	int err;
	137
	138	if (tb[NFTA_BITWISE_MASK] \|\|
	139	tb[NFTA_BITWISE_XOR])
	140	return -EINVAL;
	141
	142	if (!tb[NFTA_BITWISE_DATA])
	143	return -EINVAL;
	144
	145	err = nft_data_init(NULL, &priv->data, sizeof(priv->data), &d,
	146	tb[NFTA_BITWISE_DATA]);
	147	if (err < 0)
	148	return err;
	149	if (d.type != NFT_DATA_VALUE \|\| d.len != sizeof(u32) \|\|
	150	priv->data.data[0] >= BITS_PER_TYPE(u32)) {
	151	nft_data_release(&priv->data, d.type);
	152	return -EINVAL;
	153	}
	154
	155	return 0;
	156	}
	157
96518518 PM	158	static int nft_bitwise_init(const struct nft_ctx *ctx,
	159	const struct nft_expr *expr,
	160	const struct nlattr * const tb[])
	161	{
	162	struct nft_bitwise *priv = nft_expr_priv(expr);
36b701fa	163	u32 len;
96518518 PM	164	int err;
96518518 PM	165
36b701fa LGL	166	err = nft_parse_u32_check(tb[NFTA_BITWISE_LEN], U8_MAX, &len);
	167	if (err < 0)
	168	return err;
	169
	170	priv->len = len;
	171
4f16d25c PNA	172	err = nft_parse_register_load(tb[NFTA_BITWISE_SREG], &priv->sreg,
4f16d25c PNA	173	priv->len);
96518518 PM	174	if (err < 0)
	175	return err;
	176
345023b0 PNA	177	err = nft_parse_register_store(ctx, tb[NFTA_BITWISE_DREG],
	178	&priv->dreg, NULL, NFT_DATA_VALUE,
	179	priv->len);
96518518 PM	180	if (err < 0)
	181	return err;
	182
9d1f9799 JS	183	if (tb[NFTA_BITWISE_OP]) {
	184	priv->op = ntohl(nla_get_be32(tb[NFTA_BITWISE_OP]));
	185	switch (priv->op) {
	186	case NFT_BITWISE_BOOL:
567d746b JS	187	case NFT_BITWISE_LSHIFT:
567d746b JS	188	case NFT_BITWISE_RSHIFT:
9d1f9799 JS	189	break;
	190	default:
	191	return -EOPNOTSUPP;
	192	}
	193	} else {
	194	priv->op = NFT_BITWISE_BOOL;
	195	}
	196
3f8d9eb0 JS	197	switch(priv->op) {
	198	case NFT_BITWISE_BOOL:
	199	err = nft_bitwise_init_bool(priv, tb);
	200	break;
567d746b JS	201	case NFT_BITWISE_LSHIFT:
	202	case NFT_BITWISE_RSHIFT:
	203	err = nft_bitwise_init_shift(priv, tb);
	204	break;
71df14b0	205	}
96518518	206
71df14b0	207	return err;
96518518 PM	208	}
96518518 PM	209
4d57ca2b JS	210	static int nft_bitwise_dump_bool(struct sk_buff *skb,
	211	const struct nft_bitwise *priv)
	212	{
	213	if (nft_data_dump(skb, NFTA_BITWISE_MASK, &priv->mask,
	214	NFT_DATA_VALUE, priv->len) < 0)
	215	return -1;
	216
	217	if (nft_data_dump(skb, NFTA_BITWISE_XOR, &priv->xor,
	218	NFT_DATA_VALUE, priv->len) < 0)
	219	return -1;
	220
	221	return 0;
	222	}
	223
567d746b JS	224	static int nft_bitwise_dump_shift(struct sk_buff *skb,
	225	const struct nft_bitwise *priv)
	226	{
	227	if (nft_data_dump(skb, NFTA_BITWISE_DATA, &priv->data,
	228	NFT_DATA_VALUE, sizeof(u32)) < 0)
	229	return -1;
	230	return 0;
	231	}
	232
96518518 PM	233	static int nft_bitwise_dump(struct sk_buff skb, const struct nft_expr expr)
	234	{
	235	const struct nft_bitwise *priv = nft_expr_priv(expr);
4d57ca2b	236	int err = 0;
96518518	237
b1c96ed3	238	if (nft_dump_register(skb, NFTA_BITWISE_SREG, priv->sreg))
577c734a	239	return -1;
b1c96ed3	240	if (nft_dump_register(skb, NFTA_BITWISE_DREG, priv->dreg))
577c734a	241	return -1;
96518518	242	if (nla_put_be32(skb, NFTA_BITWISE_LEN, htonl(priv->len)))
577c734a	243	return -1;
9d1f9799 JS	244	if (nla_put_be32(skb, NFTA_BITWISE_OP, htonl(priv->op)))
9d1f9799 JS	245	return -1;
96518518	246
4d57ca2b JS	247	switch (priv->op) {
	248	case NFT_BITWISE_BOOL:
	249	err = nft_bitwise_dump_bool(skb, priv);
567d746b JS	250	break;
	251	case NFT_BITWISE_LSHIFT:
	252	case NFT_BITWISE_RSHIFT:
	253	err = nft_bitwise_dump_shift(skb, priv);
4d57ca2b JS	254	break;
4d57ca2b JS	255	}
96518518	256
4d57ca2b	257	return err;
96518518 PM	258	}
96518518 PM	259
bd8699e9 PNA	260	static struct nft_data zero;
	261
	262	static int nft_bitwise_offload(struct nft_offload_ctx *ctx,
fbf19ddf JS	263	struct nft_flow_rule *flow,
fbf19ddf JS	264	const struct nft_expr *expr)
bd8699e9 PNA	265	{
bd8699e9 PNA	266	const struct nft_bitwise *priv = nft_expr_priv(expr);
de2a6052	267	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
bd8699e9	268
ed991d43 JS	269	if (priv->op != NFT_BITWISE_BOOL)
	270	return -EOPNOTSUPP;
	271
83c156d3	272	if (memcmp(&priv->xor, &zero, sizeof(priv->xor)) \|\|
de2a6052	273	priv->sreg != priv->dreg \|\| priv->len != reg->len)
bd8699e9 PNA	274	return -EOPNOTSUPP;
bd8699e9 PNA	275
de2a6052	276	memcpy(&reg->mask, &priv->mask, sizeof(priv->mask));
bd8699e9 PNA	277
	278	return 0;
	279	}
	280
be5650f8 PNA	281	static bool nft_bitwise_reduce(struct nft_regs_track *track,
	282	const struct nft_expr *expr)
	283	{
	284	const struct nft_bitwise *priv = nft_expr_priv(expr);
	285	const struct nft_bitwise *bitwise;
34cc9e52 PNA	286	unsigned int regcount;
	287	u8 dreg;
	288	int i;
be5650f8 PNA	289
	290	if (!track->regs[priv->sreg].selector)
	291	return false;
	292
	293	bitwise = nft_expr_priv(expr);
	294	if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector &&
34cc9e52	295	track->regs[priv->sreg].num_reg == 0 &&
be5650f8 PNA	296	track->regs[priv->dreg].bitwise &&
	297	track->regs[priv->dreg].bitwise->ops == expr->ops &&
	298	priv->sreg == bitwise->sreg &&
	299	priv->dreg == bitwise->dreg &&
	300	priv->op == bitwise->op &&
	301	priv->len == bitwise->len &&
	302	!memcmp(&priv->mask, &bitwise->mask, sizeof(priv->mask)) &&
	303	!memcmp(&priv->xor, &bitwise->xor, sizeof(priv->xor)) &&
	304	!memcmp(&priv->data, &bitwise->data, sizeof(priv->data))) {
	305	track->cur = expr;
	306	return true;
	307	}
	308
34cc9e52 PNA	309	if (track->regs[priv->sreg].bitwise \|\|
	310	track->regs[priv->sreg].num_reg != 0) {
	311	nft_reg_track_cancel(track, priv->dreg, priv->len);
be5650f8 PNA	312	return false;
	313	}
	314
	315	if (priv->sreg != priv->dreg) {
34cc9e52 PNA	316	nft_reg_track_update(track, track->regs[priv->sreg].selector,
34cc9e52 PNA	317	priv->dreg, priv->len);
be5650f8	318	}
34cc9e52 PNA	319
	320	dreg = priv->dreg;
	321	regcount = DIV_ROUND_UP(priv->len, NFT_REG32_SIZE);
	322	for (i = 0; i < regcount; i++, dreg++)
	323	track->regs[priv->dreg].bitwise = expr;
be5650f8 PNA	324
	325	return false;
	326	}
	327
ef1f7df9 PM	328	static const struct nft_expr_ops nft_bitwise_ops = {
ef1f7df9 PM	329	.type = &nft_bitwise_type,
96518518	330	.size = NFT_EXPR_SIZE(sizeof(struct nft_bitwise)),
96518518 PM	331	.eval = nft_bitwise_eval,
	332	.init = nft_bitwise_init,
	333	.dump = nft_bitwise_dump,
be5650f8	334	.reduce = nft_bitwise_reduce,
bd8699e9	335	.offload = nft_bitwise_offload,
ef1f7df9 PM	336	};
ef1f7df9 PM	337
10fdd6d8 PS	338	static int
	339	nft_bitwise_extract_u32_data(const struct nlattr * const tb, u32 *out)
	340	{
	341	struct nft_data_desc desc;
	342	struct nft_data data;
	343	int err = 0;
	344
	345	err = nft_data_init(NULL, &data, sizeof(data), &desc, tb);
	346	if (err < 0)
	347	return err;
	348
	349	if (desc.type != NFT_DATA_VALUE \|\| desc.len != sizeof(u32)) {
	350	err = -EINVAL;
	351	goto err;
	352	}
	353	*out = data.data[0];
	354	err:
	355	nft_data_release(&data, desc.type);
	356	return err;
	357	}
	358
	359	static int nft_bitwise_fast_init(const struct nft_ctx *ctx,
	360	const struct nft_expr *expr,
	361	const struct nlattr * const tb[])
	362	{
	363	struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
	364	int err;
	365
4f16d25c PNA	366	err = nft_parse_register_load(tb[NFTA_BITWISE_SREG], &priv->sreg,
4f16d25c PNA	367	sizeof(u32));
10fdd6d8 PS	368	if (err < 0)
	369	return err;
	370
345023b0 PNA	371	err = nft_parse_register_store(ctx, tb[NFTA_BITWISE_DREG], &priv->dreg,
345023b0 PNA	372	NULL, NFT_DATA_VALUE, sizeof(u32));
10fdd6d8 PS	373	if (err < 0)
	374	return err;
	375
	376	if (tb[NFTA_BITWISE_DATA])
	377	return -EINVAL;
	378
	379	if (!tb[NFTA_BITWISE_MASK] \|\|
	380	!tb[NFTA_BITWISE_XOR])
	381	return -EINVAL;
	382
	383	err = nft_bitwise_extract_u32_data(tb[NFTA_BITWISE_MASK], &priv->mask);
	384	if (err < 0)
	385	return err;
	386
	387	err = nft_bitwise_extract_u32_data(tb[NFTA_BITWISE_XOR], &priv->xor);
	388	if (err < 0)
	389	return err;
	390
	391	return 0;
	392	}
	393
	394	static int
	395	nft_bitwise_fast_dump(struct sk_buff skb, const struct nft_expr expr)
	396	{
	397	const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
	398	struct nft_data data;
	399
	400	if (nft_dump_register(skb, NFTA_BITWISE_SREG, priv->sreg))
	401	return -1;
	402	if (nft_dump_register(skb, NFTA_BITWISE_DREG, priv->dreg))
	403	return -1;
	404	if (nla_put_be32(skb, NFTA_BITWISE_LEN, htonl(sizeof(u32))))
	405	return -1;
	406	if (nla_put_be32(skb, NFTA_BITWISE_OP, htonl(NFT_BITWISE_BOOL)))
	407	return -1;
	408
	409	data.data[0] = priv->mask;
	410	if (nft_data_dump(skb, NFTA_BITWISE_MASK, &data,
	411	NFT_DATA_VALUE, sizeof(u32)) < 0)
	412	return -1;
	413
	414	data.data[0] = priv->xor;
	415	if (nft_data_dump(skb, NFTA_BITWISE_XOR, &data,
	416	NFT_DATA_VALUE, sizeof(u32)) < 0)
	417	return -1;
	418
	419	return 0;
	420	}
	421
	422	static int nft_bitwise_fast_offload(struct nft_offload_ctx *ctx,
	423	struct nft_flow_rule *flow,
	424	const struct nft_expr *expr)
	425	{
	426	const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
	427	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
	428
	429	if (priv->xor \|\| priv->sreg != priv->dreg \|\| reg->len != sizeof(u32))
	430	return -EOPNOTSUPP;
	431
	432	reg->mask.data[0] = priv->mask;
	433	return 0;
	434	}
	435
be5650f8 PNA	436	static bool nft_bitwise_fast_reduce(struct nft_regs_track *track,
	437	const struct nft_expr *expr)
	438	{
	439	const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
	440	const struct nft_bitwise_fast_expr *bitwise;
	441
	442	if (!track->regs[priv->sreg].selector)
	443	return false;
	444
	445	bitwise = nft_expr_priv(expr);
	446	if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector &&
	447	track->regs[priv->dreg].bitwise &&
	448	track->regs[priv->dreg].bitwise->ops == expr->ops &&
	449	priv->sreg == bitwise->sreg &&
	450	priv->dreg == bitwise->dreg &&
	451	priv->mask == bitwise->mask &&
	452	priv->xor == bitwise->xor) {
	453	track->cur = expr;
	454	return true;
	455	}
	456
	457	if (track->regs[priv->sreg].bitwise) {
34cc9e52	458	nft_reg_track_cancel(track, priv->dreg, NFT_REG32_SIZE);
be5650f8 PNA	459	return false;
	460	}
	461
	462	if (priv->sreg != priv->dreg) {
	463	track->regs[priv->dreg].selector =
	464	track->regs[priv->sreg].selector;
	465	}
	466	track->regs[priv->dreg].bitwise = expr;
	467
	468	return false;
	469	}
	470
10fdd6d8 PS	471	const struct nft_expr_ops nft_bitwise_fast_ops = {
	472	.type = &nft_bitwise_type,
	473	.size = NFT_EXPR_SIZE(sizeof(struct nft_bitwise_fast_expr)),
	474	.eval = NULL, /* inlined */
	475	.init = nft_bitwise_fast_init,
	476	.dump = nft_bitwise_fast_dump,
be5650f8	477	.reduce = nft_bitwise_fast_reduce,
10fdd6d8 PS	478	.offload = nft_bitwise_fast_offload,
	479	};
	480
	481	static const struct nft_expr_ops *
	482	nft_bitwise_select_ops(const struct nft_ctx *ctx,
	483	const struct nlattr * const tb[])
	484	{
	485	int err;
	486	u32 len;
	487
	488	if (!tb[NFTA_BITWISE_LEN] \|\|
	489	!tb[NFTA_BITWISE_SREG] \|\|
	490	!tb[NFTA_BITWISE_DREG])
	491	return ERR_PTR(-EINVAL);
	492
	493	err = nft_parse_u32_check(tb[NFTA_BITWISE_LEN], U8_MAX, &len);
	494	if (err < 0)
	495	return ERR_PTR(err);
	496
	497	if (len != sizeof(u32))
	498	return &nft_bitwise_ops;
	499
	500	if (tb[NFTA_BITWISE_OP] &&
	501	ntohl(nla_get_be32(tb[NFTA_BITWISE_OP])) != NFT_BITWISE_BOOL)
	502	return &nft_bitwise_ops;
	503
	504	return &nft_bitwise_fast_ops;
	505	}
	506
4e24877e	507	struct nft_expr_type nft_bitwise_type __read_mostly = {
ef1f7df9	508	.name = "bitwise",
10fdd6d8	509	.select_ops = nft_bitwise_select_ops,
96518518 PM	510	.policy = nft_bitwise_policy,
96518518 PM	511	.maxattr = NFTA_BITWISE_MAX,
ef1f7df9	512	.owner = THIS_MODULE,
96518518	513	};
be5650f8 PNA	514
	515	bool nft_expr_reduce_bitwise(struct nft_regs_track *track,
	516	const struct nft_expr *expr)
	517	{
	518	const struct nft_expr *last = track->last;
	519	const struct nft_expr *next;
	520
	521	if (expr == last)
	522	return false;
	523
	524	next = nft_expr_next(expr);
	525	if (next->ops == &nft_bitwise_ops)
	526	return nft_bitwise_reduce(track, next);
	527	else if (next->ops == &nft_bitwise_fast_ops)
	528	return nft_bitwise_fast_reduce(track, next);
	529
	530	return false;
	531	}