[linux-block.git] / net / netfilter / nft_payload.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
 * Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org>
 *
 * Development of this code funded by Astaro AG (http://www.astaro.com/)
 */

#include <linux/kernel.h>
#include <linux/if_vlan.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_offload.h>
/* For layer 4 checksum field offset. */
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/icmpv6.h>
#include <linux/ip.h>
#include <linux/ipv6.h>

/* add vlan header into the user buffer for if tag was removed by offloads */
static bool
nft_payload_copy_vlan(u32 *d, const struct sk_buff *skb, u8 offset, u8 len)
{
	int mac_off = skb_mac_header(skb) - skb->data;
	u8 vlan_len, *vlanh, *dst_u8 = (u8 *) d;
	struct vlan_ethhdr veth;

	vlanh = (u8 *) &veth;
	if (offset < ETH_HLEN) {
		u8 ethlen = min_t(u8, len, ETH_HLEN - offset);

		if (skb_copy_bits(skb, mac_off, &veth, ETH_HLEN))
			return false;

		veth.h_vlan_proto = skb->vlan_proto;

		memcpy(dst_u8, vlanh + offset, ethlen);

		len -= ethlen;
		if (len == 0)
			return true;

		dst_u8 += ethlen;
		offset = ETH_HLEN;
	} else if (offset >= VLAN_ETH_HLEN) {
		offset -= VLAN_HLEN;
		goto skip;
	}

	veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb));
	veth.h_vlan_encapsulated_proto = skb->protocol;

	vlanh += offset;

	vlan_len = min_t(u8, len, VLAN_ETH_HLEN - offset);
	memcpy(dst_u8, vlanh, vlan_len);

	len -= vlan_len;
	if (!len)
		return true;

	dst_u8 += vlan_len;
 skip:
	return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0;
}

void nft_payload_eval(const struct nft_expr *expr,
		      struct nft_regs *regs,
		      const struct nft_pktinfo *pkt)
{
	const struct nft_payload *priv = nft_expr_priv(expr);
	const struct sk_buff *skb = pkt->skb;
	u32 *dest = &regs->data[priv->dreg];
	int offset;

	dest[priv->len / NFT_REG32_SIZE] = 0;
	switch (priv->base) {
	case NFT_PAYLOAD_LL_HEADER:
		if (!skb_mac_header_was_set(skb))
			goto err;

		if (skb_vlan_tag_present(skb)) {
			if (!nft_payload_copy_vlan(dest, skb,
						   priv->offset, priv->len))
				goto err;
			return;
		}
		offset = skb_mac_header(skb) - skb->data;
		break;
	case NFT_PAYLOAD_NETWORK_HEADER:
		offset = skb_network_offset(skb);
		break;
	case NFT_PAYLOAD_TRANSPORT_HEADER:
		if (!pkt->tprot_set)
			goto err;
		offset = pkt->xt.thoff;
		break;
	default:
		BUG();
	}
	offset += priv->offset;

	if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
		goto err;
	return;
err:
	regs->verdict.code = NFT_BREAK;
}

static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = {
	[NFTA_PAYLOAD_SREG]		= { .type = NLA_U32 },
	[NFTA_PAYLOAD_DREG]		= { .type = NLA_U32 },
	[NFTA_PAYLOAD_BASE]		= { .type = NLA_U32 },
	[NFTA_PAYLOAD_OFFSET]		= { .type = NLA_U32 },
	[NFTA_PAYLOAD_LEN]		= { .type = NLA_U32 },
	[NFTA_PAYLOAD_CSUM_TYPE]	= { .type = NLA_U32 },
	[NFTA_PAYLOAD_CSUM_OFFSET]	= { .type = NLA_U32 },
};

static int nft_payload_init(const struct nft_ctx *ctx,
			    const struct nft_expr *expr,
			    const struct nlattr * const tb[])
{
	struct nft_payload *priv = nft_expr_priv(expr);

	priv->base   = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
	priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
	priv->len    = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
	priv->dreg   = nft_parse_register(tb[NFTA_PAYLOAD_DREG]);

	return nft_validate_register_store(ctx, priv->dreg, NULL,
					   NFT_DATA_VALUE, priv->len);
}

static int nft_payload_dump(struct sk_buff *skb, const struct nft_expr *expr)
{
	const struct nft_payload *priv = nft_expr_priv(expr);

	if (nft_dump_register(skb, NFTA_PAYLOAD_DREG, priv->dreg) ||
	    nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
	    nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
	    nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)))
		goto nla_put_failure;
	return 0;

nla_put_failure:
	return -1;
}

static int nft_payload_offload_ll(struct nft_offload_ctx *ctx,
				  struct nft_flow_rule *flow,
				  const struct nft_payload *priv)
{
	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

	switch (priv->offset) {
	case offsetof(struct ethhdr, h_source):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
				  src, ETH_ALEN, reg);
		break;
	case offsetof(struct ethhdr, h_dest):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
				  dst, ETH_ALEN, reg);
		break;
	}

	return 0;
}

static int nft_payload_offload_ip(struct nft_offload_ctx *ctx,
				  struct nft_flow_rule *flow,
				  const struct nft_payload *priv)
{
	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

	switch (priv->offset) {
	case offsetof(struct iphdr, saddr):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src,
				  sizeof(struct in_addr), reg);
		break;
	case offsetof(struct iphdr, daddr):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst,
				  sizeof(struct in_addr), reg);
		break;
	case offsetof(struct iphdr, protocol):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
				  sizeof(__u8), reg);
		nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
		break;
	default:
		return -EOPNOTSUPP;
	}

	return 0;
}

static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx,
				  struct nft_flow_rule *flow,
				  const struct nft_payload *priv)
{
	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

	switch (priv->offset) {
	case offsetof(struct ipv6hdr, saddr):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src,
				  sizeof(struct in6_addr), reg);
		break;
	case offsetof(struct ipv6hdr, daddr):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst,
				  sizeof(struct in6_addr), reg);
		break;
	case offsetof(struct ipv6hdr, nexthdr):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
				  sizeof(__u8), reg);
		nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
		break;
	default:
		return -EOPNOTSUPP;
	}

	return 0;
}

static int nft_payload_offload_nh(struct nft_offload_ctx *ctx,
				  struct nft_flow_rule *flow,
				  const struct nft_payload *priv)
{
	int err;

	switch (ctx->dep.l3num) {
	case htons(ETH_P_IP):
		err = nft_payload_offload_ip(ctx, flow, priv);
		break;
	case htons(ETH_P_IPV6):
		err = nft_payload_offload_ip6(ctx, flow, priv);
		break;
	default:
		return -EOPNOTSUPP;
	}

	return err;
}

static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx,
				   struct nft_flow_rule *flow,
				   const struct nft_payload *priv)
{
	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

	switch (priv->offset) {
	case offsetof(struct tcphdr, source):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
				  sizeof(__be16), reg);
		break;
	case offsetof(struct tcphdr, dest):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
				  sizeof(__be16), reg);
		break;
	default:
		return -EOPNOTSUPP;
	}

	return 0;
}

static int nft_payload_offload_udp(struct nft_offload_ctx *ctx,
				   struct nft_flow_rule *flow,
				   const struct nft_payload *priv)
{
	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

	switch (priv->offset) {
	case offsetof(struct udphdr, source):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
				  sizeof(__be16), reg);
		break;
	case offsetof(struct udphdr, dest):
		NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
				  sizeof(__be16), reg);
		break;
	default:
		return -EOPNOTSUPP;
	}

	return 0;
}

static int nft_payload_offload_th(struct nft_offload_ctx *ctx,
				  struct nft_flow_rule *flow,
				  const struct nft_payload *priv)
{
	int err;

	switch (ctx->dep.protonum) {
	case IPPROTO_TCP:
		err = nft_payload_offload_tcp(ctx, flow, priv);
		break;
	case IPPROTO_UDP:
		err = nft_payload_offload_udp(ctx, flow, priv);
		break;
	default:
		return -EOPNOTSUPP;
	}

	return err;
}

static int nft_payload_offload(struct nft_offload_ctx *ctx,
			       struct nft_flow_rule *flow,
			       const struct nft_expr *expr)
{
	const struct nft_payload *priv = nft_expr_priv(expr);
	int err;

	switch (priv->base) {
	case NFT_PAYLOAD_LL_HEADER:
		err = nft_payload_offload_ll(ctx, flow, priv);
		break;
	case NFT_PAYLOAD_NETWORK_HEADER:
		err = nft_payload_offload_nh(ctx, flow, priv);
		break;
	case NFT_PAYLOAD_TRANSPORT_HEADER:
		err = nft_payload_offload_th(ctx, flow, priv);
		break;
	default:
		err = -EOPNOTSUPP;
		break;
	}
	return err;
}

static const struct nft_expr_ops nft_payload_ops = {
	.type		= &nft_payload_type,
	.size		= NFT_EXPR_SIZE(sizeof(struct nft_payload)),
	.eval		= nft_payload_eval,
	.init		= nft_payload_init,
	.dump		= nft_payload_dump,
	.offload	= nft_payload_offload,
};

const struct nft_expr_ops nft_payload_fast_ops = {
	.type		= &nft_payload_type,
	.size		= NFT_EXPR_SIZE(sizeof(struct nft_payload)),
	.eval		= nft_payload_eval,
	.init		= nft_payload_init,
	.dump		= nft_payload_dump,
	.offload	= nft_payload_offload,
};

static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum)
{
	*sum = csum_fold(csum_add(csum_sub(~csum_unfold(*sum), fsum), tsum));
	if (*sum == 0)
		*sum = CSUM_MANGLED_0;
}

static bool nft_payload_udp_checksum(struct sk_buff *skb, unsigned int thoff)
{
	struct udphdr *uh, _uh;

	uh = skb_header_pointer(skb, thoff, sizeof(_uh), &_uh);
	if (!uh)
		return false;

	return (__force bool)uh->check;
}

static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt,
				     struct sk_buff *skb,
				     unsigned int *l4csum_offset)
{
	switch (pkt->tprot) {
	case IPPROTO_TCP:
		*l4csum_offset = offsetof(struct tcphdr, check);
		break;
	case IPPROTO_UDP:
		if (!nft_payload_udp_checksum(skb, pkt->xt.thoff))
			return -1;
		/* Fall through. */
	case IPPROTO_UDPLITE:
		*l4csum_offset = offsetof(struct udphdr, check);
		break;
	case IPPROTO_ICMPV6:
		*l4csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
		break;
	default:
		return -1;
	}

	*l4csum_offset += pkt->xt.thoff;
	return 0;
}

static int nft_payload_l4csum_update(const struct nft_pktinfo *pkt,
				     struct sk_buff *skb,
				     __wsum fsum, __wsum tsum)
{
	int l4csum_offset;
	__sum16 sum;

	/* If we cannot determine layer 4 checksum offset or this packet doesn't
	 * require layer 4 checksum recalculation, skip this packet.
	 */
	if (nft_payload_l4csum_offset(pkt, skb, &l4csum_offset) < 0)
		return 0;

	if (skb_copy_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
		return -1;

	/* Checksum mangling for an arbitrary amount of bytes, based on
	 * inet_proto_csum_replace*() functions.
	 */
	if (skb->ip_summed != CHECKSUM_PARTIAL) {
		nft_csum_replace(&sum, fsum, tsum);
		if (skb->ip_summed == CHECKSUM_COMPLETE) {
			skb->csum = ~csum_add(csum_sub(~(skb->csum), fsum),
					      tsum);
		}
	} else {
		sum = ~csum_fold(csum_add(csum_sub(csum_unfold(sum), fsum),
					  tsum));
	}

	if (skb_ensure_writable(skb, l4csum_offset + sizeof(sum)) ||
	    skb_store_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
		return -1;

	return 0;
}

static int nft_payload_csum_inet(struct sk_buff *skb, const u32 *src,
				 __wsum fsum, __wsum tsum, int csum_offset)
{
	__sum16 sum;

	if (skb_copy_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
		return -1;

	nft_csum_replace(&sum, fsum, tsum);
	if (skb_ensure_writable(skb, csum_offset + sizeof(sum)) ||
	    skb_store_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
		return -1;

	return 0;
}

static void nft_payload_set_eval(const struct nft_expr *expr,
				 struct nft_regs *regs,
				 const struct nft_pktinfo *pkt)
{
	const struct nft_payload_set *priv = nft_expr_priv(expr);
	struct sk_buff *skb = pkt->skb;
	const u32 *src = &regs->data[priv->sreg];
	int offset, csum_offset;
	__wsum fsum, tsum;

	switch (priv->base) {
	case NFT_PAYLOAD_LL_HEADER:
		if (!skb_mac_header_was_set(skb))
			goto err;
		offset = skb_mac_header(skb) - skb->data;
		break;
	case NFT_PAYLOAD_NETWORK_HEADER:
		offset = skb_network_offset(skb);
		break;
	case NFT_PAYLOAD_TRANSPORT_HEADER:
		if (!pkt->tprot_set)
			goto err;
		offset = pkt->xt.thoff;
		break;
	default:
		BUG();
	}

	csum_offset = offset + priv->csum_offset;
	offset += priv->offset;

	if ((priv->csum_type == NFT_PAYLOAD_CSUM_INET || priv->csum_flags) &&
	    (priv->base != NFT_PAYLOAD_TRANSPORT_HEADER ||
	     skb->ip_summed != CHECKSUM_PARTIAL)) {
		fsum = skb_checksum(skb, offset, priv->len, 0);
		tsum = csum_partial(src, priv->len, 0);

		if (priv->csum_type == NFT_PAYLOAD_CSUM_INET &&
		    nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset))
			goto err;

		if (priv->csum_flags &&
		    nft_payload_l4csum_update(pkt, skb, fsum, tsum) < 0)
			goto err;
	}

	if (skb_ensure_writable(skb, max(offset + priv->len, 0)) ||
	    skb_store_bits(skb, offset, src, priv->len) < 0)
		goto err;

	return;
err:
	regs->verdict.code = NFT_BREAK;
}

static int nft_payload_set_init(const struct nft_ctx *ctx,
				const struct nft_expr *expr,
				const struct nlattr * const tb[])
{
	struct nft_payload_set *priv = nft_expr_priv(expr);

	priv->base        = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
	priv->offset      = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
	priv->len         = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
	priv->sreg        = nft_parse_register(tb[NFTA_PAYLOAD_SREG]);

	if (tb[NFTA_PAYLOAD_CSUM_TYPE])
		priv->csum_type =
			ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE]));
	if (tb[NFTA_PAYLOAD_CSUM_OFFSET])
		priv->csum_offset =
			ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_OFFSET]));
	if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) {
		u32 flags;

		flags = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_FLAGS]));
		if (flags & ~NFT_PAYLOAD_L4CSUM_PSEUDOHDR)
			return -EINVAL;

		priv->csum_flags = flags;
	}

	switch (priv->csum_type) {
	case NFT_PAYLOAD_CSUM_NONE:
	case NFT_PAYLOAD_CSUM_INET:
		break;
	default:
		return -EOPNOTSUPP;
	}

	return nft_validate_register_load(priv->sreg, priv->len);
}

static int nft_payload_set_dump(struct sk_buff *skb, const struct nft_expr *expr)
{
	const struct nft_payload_set *priv = nft_expr_priv(expr);

	if (nft_dump_register(skb, NFTA_PAYLOAD_SREG, priv->sreg) ||
	    nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
	    nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
	    nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)) ||
	    nla_put_be32(skb, NFTA_PAYLOAD_CSUM_TYPE, htonl(priv->csum_type)) ||
	    nla_put_be32(skb, NFTA_PAYLOAD_CSUM_OFFSET,
			 htonl(priv->csum_offset)) ||
	    nla_put_be32(skb, NFTA_PAYLOAD_CSUM_FLAGS, htonl(priv->csum_flags)))
		goto nla_put_failure;
	return 0;

nla_put_failure:
	return -1;
}

static const struct nft_expr_ops nft_payload_set_ops = {
	.type		= &nft_payload_type,
	.size		= NFT_EXPR_SIZE(sizeof(struct nft_payload_set)),
	.eval		= nft_payload_set_eval,
	.init		= nft_payload_set_init,
	.dump		= nft_payload_set_dump,
};

static const struct nft_expr_ops *
nft_payload_select_ops(const struct nft_ctx *ctx,
		       const struct nlattr * const tb[])
{
	enum nft_payload_bases base;
	unsigned int offset, len;

	if (tb[NFTA_PAYLOAD_BASE] == NULL ||
	    tb[NFTA_PAYLOAD_OFFSET] == NULL ||
	    tb[NFTA_PAYLOAD_LEN] == NULL)
		return ERR_PTR(-EINVAL);

	base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
	switch (base) {
	case NFT_PAYLOAD_LL_HEADER:
	case NFT_PAYLOAD_NETWORK_HEADER:
	case NFT_PAYLOAD_TRANSPORT_HEADER:
		break;
	default:
		return ERR_PTR(-EOPNOTSUPP);
	}

	if (tb[NFTA_PAYLOAD_SREG] != NULL) {
		if (tb[NFTA_PAYLOAD_DREG] != NULL)
			return ERR_PTR(-EINVAL);
		return &nft_payload_set_ops;
	}

	if (tb[NFTA_PAYLOAD_DREG] == NULL)
		return ERR_PTR(-EINVAL);

	offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
	len    = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));

	if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) &&
	    base != NFT_PAYLOAD_LL_HEADER)
		return &nft_payload_fast_ops;
	else
		return &nft_payload_ops;
}

struct nft_expr_type nft_payload_type __read_mostly = {
	.name		= "payload",
	.select_ops	= nft_payload_select_ops,
	.policy		= nft_payload_policy,
	.maxattr	= NFTA_PAYLOAD_MAX,
	.owner		= THIS_MODULE,
};
Commit	Line	Data
d2912cb1	1	// SPDX-License-Identifier: GPL-2.0-only
96518518	2	/*
ef1f7df9	3	* Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
18140969	4	* Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org>
96518518	5	*
96518518 PM	6	* Development of this code funded by Astaro AG (http://www.astaro.com/)
	7	*/
	8
	9	#include <linux/kernel.h>
8cfd23e6	10	#include <linux/if_vlan.h>
96518518 PM	11	#include <linux/init.h>
	12	#include <linux/module.h>
	13	#include <linux/netlink.h>
	14	#include <linux/netfilter.h>
	15	#include <linux/netfilter/nf_tables.h>
	16	#include <net/netfilter/nf_tables_core.h>
	17	#include <net/netfilter/nf_tables.h>
c9626a2c	18	#include <net/netfilter/nf_tables_offload.h>
18140969 PNA	19	/* For layer 4 checksum field offset. */
	20	#include <linux/tcp.h>
	21	#include <linux/udp.h>
	22	#include <linux/icmpv6.h>
c9626a2c PNA	23	#include <linux/ip.h>
c9626a2c PNA	24	#include <linux/ipv6.h>
96518518	25
8cfd23e6 FW	26	/* add vlan header into the user buffer for if tag was removed by offloads */
	27	static bool
	28	nft_payload_copy_vlan(u32 d, const struct sk_buff skb, u8 offset, u8 len)
	29	{
	30	int mac_off = skb_mac_header(skb) - skb->data;
	31	u8 vlan_len, vlanh, dst_u8 = (u8 *) d;
	32	struct vlan_ethhdr veth;
	33
	34	vlanh = (u8 *) &veth;
	35	if (offset < ETH_HLEN) {
	36	u8 ethlen = min_t(u8, len, ETH_HLEN - offset);
	37
	38	if (skb_copy_bits(skb, mac_off, &veth, ETH_HLEN))
	39	return false;
	40
	41	veth.h_vlan_proto = skb->vlan_proto;
	42
	43	memcpy(dst_u8, vlanh + offset, ethlen);
	44
	45	len -= ethlen;
	46	if (len == 0)
	47	return true;
	48
	49	dst_u8 += ethlen;
	50	offset = ETH_HLEN;
	51	} else if (offset >= VLAN_ETH_HLEN) {
	52	offset -= VLAN_HLEN;
	53	goto skip;
	54	}
	55
	56	veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb));
	57	veth.h_vlan_encapsulated_proto = skb->protocol;
	58
	59	vlanh += offset;
	60
	61	vlan_len = min_t(u8, len, VLAN_ETH_HLEN - offset);
	62	memcpy(dst_u8, vlanh, vlan_len);
	63
	64	len -= vlan_len;
	65	if (!len)
	66	return true;
	67
	68	dst_u8 += vlan_len;
	69	skip:
	70	return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0;
	71	}
	72
10870dd8 FW	73	void nft_payload_eval(const struct nft_expr *expr,
	74	struct nft_regs *regs,
	75	const struct nft_pktinfo *pkt)
96518518 PM	76	{
	77	const struct nft_payload *priv = nft_expr_priv(expr);
	78	const struct sk_buff *skb = pkt->skb;
49499c3e	79	u32 *dest = &regs->data[priv->dreg];
96518518 PM	80	int offset;
96518518 PM	81
8cfd23e6	82	dest[priv->len / NFT_REG32_SIZE] = 0;
96518518 PM	83	switch (priv->base) {
	84	case NFT_PAYLOAD_LL_HEADER:
	85	if (!skb_mac_header_was_set(skb))
	86	goto err;
8cfd23e6 FW	87
	88	if (skb_vlan_tag_present(skb)) {
	89	if (!nft_payload_copy_vlan(dest, skb,
	90	priv->offset, priv->len))
	91	goto err;
	92	return;
	93	}
96518518 PM	94	offset = skb_mac_header(skb) - skb->data;
	95	break;
	96	case NFT_PAYLOAD_NETWORK_HEADER:
	97	offset = skb_network_offset(skb);
	98	break;
	99	case NFT_PAYLOAD_TRANSPORT_HEADER:
a20877b5 LZ	100	if (!pkt->tprot_set)
a20877b5 LZ	101	goto err;
c54032e0	102	offset = pkt->xt.thoff;
96518518 PM	103	break;
	104	default:
	105	BUG();
	106	}
	107	offset += priv->offset;
	108
fad136ea	109	if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
96518518 PM	110	goto err;
	111	return;
	112	err:
a55e22e9	113	regs->verdict.code = NFT_BREAK;
96518518 PM	114	}
	115
	116	static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = {
7ec3f7b4 PM	117	[NFTA_PAYLOAD_SREG] = { .type = NLA_U32 },
	118	[NFTA_PAYLOAD_DREG] = { .type = NLA_U32 },
	119	[NFTA_PAYLOAD_BASE] = { .type = NLA_U32 },
	120	[NFTA_PAYLOAD_OFFSET] = { .type = NLA_U32 },
	121	[NFTA_PAYLOAD_LEN] = { .type = NLA_U32 },
	122	[NFTA_PAYLOAD_CSUM_TYPE] = { .type = NLA_U32 },
	123	[NFTA_PAYLOAD_CSUM_OFFSET] = { .type = NLA_U32 },
96518518 PM	124	};
	125
	126	static int nft_payload_init(const struct nft_ctx *ctx,
	127	const struct nft_expr *expr,
	128	const struct nlattr * const tb[])
	129	{
	130	struct nft_payload *priv = nft_expr_priv(expr);
96518518	131
c29b72e0	132	priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
96518518 PM	133	priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
96518518 PM	134	priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
b1c96ed3	135	priv->dreg = nft_parse_register(tb[NFTA_PAYLOAD_DREG]);
96518518	136
1ec10212 PM	137	return nft_validate_register_store(ctx, priv->dreg, NULL,
1ec10212 PM	138	NFT_DATA_VALUE, priv->len);
96518518 PM	139	}
	140
	141	static int nft_payload_dump(struct sk_buff skb, const struct nft_expr expr)
	142	{
	143	const struct nft_payload *priv = nft_expr_priv(expr);
	144
b1c96ed3	145	if (nft_dump_register(skb, NFTA_PAYLOAD_DREG, priv->dreg) \|\|
96518518 PM	146	nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) \|\|
	147	nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) \|\|
	148	nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)))
	149	goto nla_put_failure;
	150	return 0;
	151
	152	nla_put_failure:
	153	return -1;
	154	}
	155
c9626a2c PNA	156	static int nft_payload_offload_ll(struct nft_offload_ctx *ctx,
	157	struct nft_flow_rule *flow,
	158	const struct nft_payload *priv)
	159	{
	160	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
	161
	162	switch (priv->offset) {
	163	case offsetof(struct ethhdr, h_source):
	164	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
	165	src, ETH_ALEN, reg);
	166	break;
	167	case offsetof(struct ethhdr, h_dest):
	168	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
	169	dst, ETH_ALEN, reg);
	170	break;
	171	}
	172
	173	return 0;
	174	}
	175
	176	static int nft_payload_offload_ip(struct nft_offload_ctx *ctx,
	177	struct nft_flow_rule *flow,
	178	const struct nft_payload *priv)
	179	{
	180	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
	181
	182	switch (priv->offset) {
	183	case offsetof(struct iphdr, saddr):
	184	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src,
	185	sizeof(struct in_addr), reg);
	186	break;
	187	case offsetof(struct iphdr, daddr):
	188	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst,
	189	sizeof(struct in_addr), reg);
	190	break;
	191	case offsetof(struct iphdr, protocol):
	192	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
	193	sizeof(__u8), reg);
	194	nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
	195	break;
	196	default:
	197	return -EOPNOTSUPP;
	198	}
	199
	200	return 0;
	201	}
	202
	203	static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx,
	204	struct nft_flow_rule *flow,
	205	const struct nft_payload *priv)
	206	{
	207	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
	208
	209	switch (priv->offset) {
	210	case offsetof(struct ipv6hdr, saddr):
	211	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src,
	212	sizeof(struct in6_addr), reg);
	213	break;
	214	case offsetof(struct ipv6hdr, daddr):
	215	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst,
	216	sizeof(struct in6_addr), reg);
	217	break;
	218	case offsetof(struct ipv6hdr, nexthdr):
	219	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
220	sizeof(__u8), reg);
221	nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
222	break;
223	default:
224	return -EOPNOTSUPP;
225	}
226
227	return 0;
228	}
229
230	static int nft_payload_offload_nh(struct nft_offload_ctx *ctx,
231	struct nft_flow_rule *flow,
232	const struct nft_payload *priv)
233	{
234	int err;
235
236	switch (ctx->dep.l3num) {
237	case htons(ETH_P_IP):
238	err = nft_payload_offload_ip(ctx, flow, priv);
239	break;
240	case htons(ETH_P_IPV6):
241	err = nft_payload_offload_ip6(ctx, flow, priv);
242	break;
243	default:
244	return -EOPNOTSUPP;
245	}
246
247	return err;
248	}
249
250	static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx,
251	struct nft_flow_rule *flow,
252	const struct nft_payload *priv)
253	{
254	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
255
256	switch (priv->offset) {
257	case offsetof(struct tcphdr, source):
258	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
259	sizeof(__be16), reg);
260	break;
261	case offsetof(struct tcphdr, dest):
262	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
263	sizeof(__be16), reg);
264	break;
265	default:
266	return -EOPNOTSUPP;
267	}
268
269	return 0;
270	}
271
272	static int nft_payload_offload_udp(struct nft_offload_ctx *ctx,
273	struct nft_flow_rule *flow,
274	const struct nft_payload *priv)
275	{
276	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
277
278	switch (priv->offset) {
279	case offsetof(struct udphdr, source):
280	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
281	sizeof(__be16), reg);
282	break;
283	case offsetof(struct udphdr, dest):
284	NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
285	sizeof(__be16), reg);
286	break;
287	default:
288	return -EOPNOTSUPP;
289	}
290
291	return 0;
292	}
293
294	static int nft_payload_offload_th(struct nft_offload_ctx *ctx,
295	struct nft_flow_rule *flow,
296	const struct nft_payload *priv)
297	{
298	int err;
299
300	switch (ctx->dep.protonum) {
301	case IPPROTO_TCP:
302	err = nft_payload_offload_tcp(ctx, flow, priv);
303	break;
304	case IPPROTO_UDP:
305	err = nft_payload_offload_udp(ctx, flow, priv);
306	break;
307	default:
308	return -EOPNOTSUPP;
309	}
310
311	return err;
312	}
313
314	static int nft_payload_offload(struct nft_offload_ctx *ctx,
315	struct nft_flow_rule *flow,
316	const struct nft_expr *expr)
317	{
318	const struct nft_payload *priv = nft_expr_priv(expr);
319	int err;
320
321	switch (priv->base) {
322	case NFT_PAYLOAD_LL_HEADER:
323	err = nft_payload_offload_ll(ctx, flow, priv);
324	break;
325	case NFT_PAYLOAD_NETWORK_HEADER:
326	err = nft_payload_offload_nh(ctx, flow, priv);
327	break;
328	case NFT_PAYLOAD_TRANSPORT_HEADER:
329	err = nft_payload_offload_th(ctx, flow, priv);
330	break;
331	default:
332	err = -EOPNOTSUPP;
333	break;
334	}
335	return err;
336	}
337
ef1f7df9 PM	338	static const struct nft_expr_ops nft_payload_ops = {
ef1f7df9 PM	339	.type = &nft_payload_type,
96518518	340	.size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
96518518 PM	341	.eval = nft_payload_eval,
	342	.init = nft_payload_init,
	343	.dump = nft_payload_dump,
c9626a2c	344	.offload = nft_payload_offload,
ef1f7df9 PM	345	};
ef1f7df9 PM	346
c29b72e0 PM	347	const struct nft_expr_ops nft_payload_fast_ops = {
	348	.type = &nft_payload_type,
	349	.size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
	350	.eval = nft_payload_eval,
	351	.init = nft_payload_init,
	352	.dump = nft_payload_dump,
c9626a2c	353	.offload = nft_payload_offload,
c29b72e0 PM	354	};
c29b72e0 PM	355
18140969 PNA	356	static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum)
	357	{
	358	sum = csum_fold(csum_add(csum_sub(~csum_unfold(sum), fsum), tsum));
	359	if (*sum == 0)
	360	*sum = CSUM_MANGLED_0;
	361	}
	362
	363	static bool nft_payload_udp_checksum(struct sk_buff *skb, unsigned int thoff)
	364	{
	365	struct udphdr *uh, _uh;
	366
	367	uh = skb_header_pointer(skb, thoff, sizeof(_uh), &_uh);
	368	if (!uh)
	369	return false;
	370
5fd02ebe	371	return (__force bool)uh->check;
18140969 PNA	372	}
	373
	374	static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt,
	375	struct sk_buff *skb,
	376	unsigned int *l4csum_offset)
	377	{
	378	switch (pkt->tprot) {
	379	case IPPROTO_TCP:
	380	*l4csum_offset = offsetof(struct tcphdr, check);
	381	break;
	382	case IPPROTO_UDP:
	383	if (!nft_payload_udp_checksum(skb, pkt->xt.thoff))
	384	return -1;
	385	/* Fall through. */
	386	case IPPROTO_UDPLITE:
	387	*l4csum_offset = offsetof(struct udphdr, check);
	388	break;
	389	case IPPROTO_ICMPV6:
	390	*l4csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
	391	break;
	392	default:
	393	return -1;
	394	}
	395
	396	*l4csum_offset += pkt->xt.thoff;
	397	return 0;
	398	}
	399
	400	static int nft_payload_l4csum_update(const struct nft_pktinfo *pkt,
	401	struct sk_buff *skb,
	402	__wsum fsum, __wsum tsum)
	403	{
	404	int l4csum_offset;
	405	__sum16 sum;
	406
	407	/* If we cannot determine layer 4 checksum offset or this packet doesn't
	408	* require layer 4 checksum recalculation, skip this packet.
	409	*/
	410	if (nft_payload_l4csum_offset(pkt, skb, &l4csum_offset) < 0)
	411	return 0;
	412
	413	if (skb_copy_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
	414	return -1;
	415
	416	/* Checksum mangling for an arbitrary amount of bytes, based on
	417	* inet_proto_csum_replace*() functions.
	418	*/
	419	if (skb->ip_summed != CHECKSUM_PARTIAL) {
	420	nft_csum_replace(&sum, fsum, tsum);
	421	if (skb->ip_summed == CHECKSUM_COMPLETE) {
	422	skb->csum = ~csum_add(csum_sub(~(skb->csum), fsum),
	423	tsum);
	424	}
	425	} else {
	426	sum = ~csum_fold(csum_add(csum_sub(csum_unfold(sum), fsum),
	427	tsum));
	428	}
	429
7418ee4c	430	if (skb_ensure_writable(skb, l4csum_offset + sizeof(sum)) \|\|
18140969 PNA	431	skb_store_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
	432	return -1;
	433
	434	return 0;
	435	}
	436
053d20f5 PNA	437	static int nft_payload_csum_inet(struct sk_buff skb, const u32 src,
	438	__wsum fsum, __wsum tsum, int csum_offset)
	439	{
	440	__sum16 sum;
	441
	442	if (skb_copy_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
	443	return -1;
	444
	445	nft_csum_replace(&sum, fsum, tsum);
7418ee4c	446	if (skb_ensure_writable(skb, csum_offset + sizeof(sum)) \|\|
053d20f5 PNA	447	skb_store_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
	448	return -1;
	449
	450	return 0;
	451	}
	452
7ec3f7b4 PM	453	static void nft_payload_set_eval(const struct nft_expr *expr,
	454	struct nft_regs *regs,
	455	const struct nft_pktinfo *pkt)
	456	{
	457	const struct nft_payload_set *priv = nft_expr_priv(expr);
	458	struct sk_buff *skb = pkt->skb;
	459	const u32 *src = &regs->data[priv->sreg];
	460	int offset, csum_offset;
	461	__wsum fsum, tsum;
7ec3f7b4 PM	462
	463	switch (priv->base) {
	464	case NFT_PAYLOAD_LL_HEADER:
	465	if (!skb_mac_header_was_set(skb))
	466	goto err;
	467	offset = skb_mac_header(skb) - skb->data;
	468	break;
	469	case NFT_PAYLOAD_NETWORK_HEADER:
	470	offset = skb_network_offset(skb);
	471	break;
	472	case NFT_PAYLOAD_TRANSPORT_HEADER:
a20877b5 LZ	473	if (!pkt->tprot_set)
a20877b5 LZ	474	goto err;
7ec3f7b4 PM	475	offset = pkt->xt.thoff;
	476	break;
	477	default:
	478	BUG();
	479	}
	480
	481	csum_offset = offset + priv->csum_offset;
	482	offset += priv->offset;
	483
053d20f5	484	if ((priv->csum_type == NFT_PAYLOAD_CSUM_INET \|\| priv->csum_flags) &&
7ec3f7b4 PM	485	(priv->base != NFT_PAYLOAD_TRANSPORT_HEADER \|\|
7ec3f7b4 PM	486	skb->ip_summed != CHECKSUM_PARTIAL)) {
7ec3f7b4 PM	487	fsum = skb_checksum(skb, offset, priv->len, 0);
7ec3f7b4 PM	488	tsum = csum_partial(src, priv->len, 0);
7ec3f7b4	489
053d20f5 PNA	490	if (priv->csum_type == NFT_PAYLOAD_CSUM_INET &&
053d20f5 PNA	491	nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset))
7ec3f7b4	492	goto err;
18140969 PNA	493
	494	if (priv->csum_flags &&
	495	nft_payload_l4csum_update(pkt, skb, fsum, tsum) < 0)
	496	goto err;
7ec3f7b4 PM	497	}
7ec3f7b4 PM	498
7418ee4c	499	if (skb_ensure_writable(skb, max(offset + priv->len, 0)) \|\|
7ec3f7b4 PM	500	skb_store_bits(skb, offset, src, priv->len) < 0)
	501	goto err;
	502
	503	return;
	504	err:
	505	regs->verdict.code = NFT_BREAK;
	506	}
	507
	508	static int nft_payload_set_init(const struct nft_ctx *ctx,
	509	const struct nft_expr *expr,
	510	const struct nlattr * const tb[])
	511	{
	512	struct nft_payload_set *priv = nft_expr_priv(expr);
	513
	514	priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
	515	priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
	516	priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
	517	priv->sreg = nft_parse_register(tb[NFTA_PAYLOAD_SREG]);
	518
	519	if (tb[NFTA_PAYLOAD_CSUM_TYPE])
	520	priv->csum_type =
	521	ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE]));
	522	if (tb[NFTA_PAYLOAD_CSUM_OFFSET])
	523	priv->csum_offset =
	524	ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_OFFSET]));
18140969 PNA	525	if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) {
	526	u32 flags;
	527
	528	flags = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_FLAGS]));
	529	if (flags & ~NFT_PAYLOAD_L4CSUM_PSEUDOHDR)
	530	return -EINVAL;
	531
	532	priv->csum_flags = flags;
	533	}
7ec3f7b4 PM	534
	535	switch (priv->csum_type) {
	536	case NFT_PAYLOAD_CSUM_NONE:
	537	case NFT_PAYLOAD_CSUM_INET:
	538	break;
	539	default:
	540	return -EOPNOTSUPP;
	541	}
	542
	543	return nft_validate_register_load(priv->sreg, priv->len);
	544	}
	545
	546	static int nft_payload_set_dump(struct sk_buff skb, const struct nft_expr expr)
	547	{
	548	const struct nft_payload_set *priv = nft_expr_priv(expr);
	549
	550	if (nft_dump_register(skb, NFTA_PAYLOAD_SREG, priv->sreg) \|\|
	551	nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) \|\|
	552	nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) \|\|
	553	nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)) \|\|
	554	nla_put_be32(skb, NFTA_PAYLOAD_CSUM_TYPE, htonl(priv->csum_type)) \|\|
	555	nla_put_be32(skb, NFTA_PAYLOAD_CSUM_OFFSET,
18140969 PNA	556	htonl(priv->csum_offset)) \|\|
18140969 PNA	557	nla_put_be32(skb, NFTA_PAYLOAD_CSUM_FLAGS, htonl(priv->csum_flags)))
7ec3f7b4 PM	558	goto nla_put_failure;
	559	return 0;
	560
	561	nla_put_failure:
	562	return -1;
	563	}
	564
	565	static const struct nft_expr_ops nft_payload_set_ops = {
	566	.type = &nft_payload_type,
	567	.size = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)),
	568	.eval = nft_payload_set_eval,
	569	.init = nft_payload_set_init,
	570	.dump = nft_payload_set_dump,
	571	};
	572
0ca743a5 PNA	573	static const struct nft_expr_ops *
	574	nft_payload_select_ops(const struct nft_ctx *ctx,
	575	const struct nlattr * const tb[])
c29b72e0 PM	576	{
	577	enum nft_payload_bases base;
	578	unsigned int offset, len;
	579
7ec3f7b4	580	if (tb[NFTA_PAYLOAD_BASE] == NULL \|\|
c29b72e0 PM	581	tb[NFTA_PAYLOAD_OFFSET] == NULL \|\|
	582	tb[NFTA_PAYLOAD_LEN] == NULL)
	583	return ERR_PTR(-EINVAL);
	584
	585	base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
	586	switch (base) {
	587	case NFT_PAYLOAD_LL_HEADER:
	588	case NFT_PAYLOAD_NETWORK_HEADER:
	589	case NFT_PAYLOAD_TRANSPORT_HEADER:
	590	break;
	591	default:
	592	return ERR_PTR(-EOPNOTSUPP);
	593	}
	594
7ec3f7b4 PM	595	if (tb[NFTA_PAYLOAD_SREG] != NULL) {
	596	if (tb[NFTA_PAYLOAD_DREG] != NULL)
	597	return ERR_PTR(-EINVAL);
	598	return &nft_payload_set_ops;
	599	}
	600
	601	if (tb[NFTA_PAYLOAD_DREG] == NULL)
	602	return ERR_PTR(-EINVAL);
	603
c29b72e0	604	offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
45d9bcda	605	len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
c29b72e0	606
f627ed91 NA	607	if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) &&
f627ed91 NA	608	base != NFT_PAYLOAD_LL_HEADER)
c29b72e0 PM	609	return &nft_payload_fast_ops;
	610	else
	611	return &nft_payload_ops;
	612	}
	613
4e24877e	614	struct nft_expr_type nft_payload_type __read_mostly = {
ef1f7df9	615	.name = "payload",
c29b72e0	616	.select_ops = nft_payload_select_ops,
96518518 PM	617	.policy = nft_payload_policy,
96518518 PM	618	.maxattr = NFTA_PAYLOAD_MAX,
ef1f7df9	619	.owner = THIS_MODULE,
96518518	620	};