1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
4 * Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org>
6 * Development of this code funded by Astaro AG (http://www.astaro.com/)
9 #include <linux/kernel.h>
10 #include <linux/if_vlan.h>
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/netlink.h>
14 #include <linux/netfilter.h>
15 #include <linux/netfilter/nf_tables.h>
16 #include <net/netfilter/nf_tables_core.h>
17 #include <net/netfilter/nf_tables.h>
18 #include <net/netfilter/nf_tables_offload.h>
19 /* For layer 4 checksum field offset. */
20 #include <linux/tcp.h>
21 #include <linux/udp.h>
22 #include <linux/icmpv6.h>
24 #include <linux/ipv6.h>
26 /* add vlan header into the user buffer for if tag was removed by offloads */
28 nft_payload_copy_vlan(u32 *d, const struct sk_buff *skb, u8 offset, u8 len)
30 int mac_off = skb_mac_header(skb) - skb->data;
31 u8 vlan_len, *vlanh, *dst_u8 = (u8 *) d;
32 struct vlan_ethhdr veth;
35 if (offset < ETH_HLEN) {
36 u8 ethlen = min_t(u8, len, ETH_HLEN - offset);
38 if (skb_copy_bits(skb, mac_off, &veth, ETH_HLEN))
41 veth.h_vlan_proto = skb->vlan_proto;
43 memcpy(dst_u8, vlanh + offset, ethlen);
51 } else if (offset >= VLAN_ETH_HLEN) {
56 veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb));
57 veth.h_vlan_encapsulated_proto = skb->protocol;
61 vlan_len = min_t(u8, len, VLAN_ETH_HLEN - offset);
62 memcpy(dst_u8, vlanh, vlan_len);
70 return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0;
73 void nft_payload_eval(const struct nft_expr *expr,
74 struct nft_regs *regs,
75 const struct nft_pktinfo *pkt)
77 const struct nft_payload *priv = nft_expr_priv(expr);
78 const struct sk_buff *skb = pkt->skb;
79 u32 *dest = ®s->data[priv->dreg];
82 dest[priv->len / NFT_REG32_SIZE] = 0;
84 case NFT_PAYLOAD_LL_HEADER:
85 if (!skb_mac_header_was_set(skb))
88 if (skb_vlan_tag_present(skb)) {
89 if (!nft_payload_copy_vlan(dest, skb,
90 priv->offset, priv->len))
94 offset = skb_mac_header(skb) - skb->data;
96 case NFT_PAYLOAD_NETWORK_HEADER:
97 offset = skb_network_offset(skb);
99 case NFT_PAYLOAD_TRANSPORT_HEADER:
102 offset = pkt->xt.thoff;
107 offset += priv->offset;
109 if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
113 regs->verdict.code = NFT_BREAK;
116 static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = {
117 [NFTA_PAYLOAD_SREG] = { .type = NLA_U32 },
118 [NFTA_PAYLOAD_DREG] = { .type = NLA_U32 },
119 [NFTA_PAYLOAD_BASE] = { .type = NLA_U32 },
120 [NFTA_PAYLOAD_OFFSET] = { .type = NLA_U32 },
121 [NFTA_PAYLOAD_LEN] = { .type = NLA_U32 },
122 [NFTA_PAYLOAD_CSUM_TYPE] = { .type = NLA_U32 },
123 [NFTA_PAYLOAD_CSUM_OFFSET] = { .type = NLA_U32 },
126 static int nft_payload_init(const struct nft_ctx *ctx,
127 const struct nft_expr *expr,
128 const struct nlattr * const tb[])
130 struct nft_payload *priv = nft_expr_priv(expr);
132 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
133 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
134 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
135 priv->dreg = nft_parse_register(tb[NFTA_PAYLOAD_DREG]);
137 return nft_validate_register_store(ctx, priv->dreg, NULL,
138 NFT_DATA_VALUE, priv->len);
141 static int nft_payload_dump(struct sk_buff *skb, const struct nft_expr *expr)
143 const struct nft_payload *priv = nft_expr_priv(expr);
145 if (nft_dump_register(skb, NFTA_PAYLOAD_DREG, priv->dreg) ||
146 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
147 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
148 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)))
149 goto nla_put_failure;
156 static int nft_payload_offload_ll(struct nft_offload_ctx *ctx,
157 struct nft_flow_rule *flow,
158 const struct nft_payload *priv)
160 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
162 switch (priv->offset) {
163 case offsetof(struct ethhdr, h_source):
164 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
167 case offsetof(struct ethhdr, h_dest):
168 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
176 static int nft_payload_offload_ip(struct nft_offload_ctx *ctx,
177 struct nft_flow_rule *flow,
178 const struct nft_payload *priv)
180 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
182 switch (priv->offset) {
183 case offsetof(struct iphdr, saddr):
184 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src,
185 sizeof(struct in_addr), reg);
187 case offsetof(struct iphdr, daddr):
188 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst,
189 sizeof(struct in_addr), reg);
191 case offsetof(struct iphdr, protocol):
192 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
194 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
203 static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx,
204 struct nft_flow_rule *flow,
205 const struct nft_payload *priv)
207 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
209 switch (priv->offset) {
210 case offsetof(struct ipv6hdr, saddr):
211 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src,
212 sizeof(struct in6_addr), reg);
214 case offsetof(struct ipv6hdr, daddr):
215 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst,
216 sizeof(struct in6_addr), reg);
218 case offsetof(struct ipv6hdr, nexthdr):
219 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
221 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
230 static int nft_payload_offload_nh(struct nft_offload_ctx *ctx,
231 struct nft_flow_rule *flow,
232 const struct nft_payload *priv)
236 switch (ctx->dep.l3num) {
237 case htons(ETH_P_IP):
238 err = nft_payload_offload_ip(ctx, flow, priv);
240 case htons(ETH_P_IPV6):
241 err = nft_payload_offload_ip6(ctx, flow, priv);
250 static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx,
251 struct nft_flow_rule *flow,
252 const struct nft_payload *priv)
254 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
256 switch (priv->offset) {
257 case offsetof(struct tcphdr, source):
258 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
259 sizeof(__be16), reg);
261 case offsetof(struct tcphdr, dest):
262 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
263 sizeof(__be16), reg);
272 static int nft_payload_offload_udp(struct nft_offload_ctx *ctx,
273 struct nft_flow_rule *flow,
274 const struct nft_payload *priv)
276 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
278 switch (priv->offset) {
279 case offsetof(struct udphdr, source):
280 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
281 sizeof(__be16), reg);
283 case offsetof(struct udphdr, dest):
284 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
285 sizeof(__be16), reg);
294 static int nft_payload_offload_th(struct nft_offload_ctx *ctx,
295 struct nft_flow_rule *flow,
296 const struct nft_payload *priv)
300 switch (ctx->dep.protonum) {
302 err = nft_payload_offload_tcp(ctx, flow, priv);
305 err = nft_payload_offload_udp(ctx, flow, priv);
314 static int nft_payload_offload(struct nft_offload_ctx *ctx,
315 struct nft_flow_rule *flow,
316 const struct nft_expr *expr)
318 const struct nft_payload *priv = nft_expr_priv(expr);
321 switch (priv->base) {
322 case NFT_PAYLOAD_LL_HEADER:
323 err = nft_payload_offload_ll(ctx, flow, priv);
325 case NFT_PAYLOAD_NETWORK_HEADER:
326 err = nft_payload_offload_nh(ctx, flow, priv);
328 case NFT_PAYLOAD_TRANSPORT_HEADER:
329 err = nft_payload_offload_th(ctx, flow, priv);
338 static const struct nft_expr_ops nft_payload_ops = {
339 .type = &nft_payload_type,
340 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
341 .eval = nft_payload_eval,
342 .init = nft_payload_init,
343 .dump = nft_payload_dump,
344 .offload = nft_payload_offload,
347 const struct nft_expr_ops nft_payload_fast_ops = {
348 .type = &nft_payload_type,
349 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
350 .eval = nft_payload_eval,
351 .init = nft_payload_init,
352 .dump = nft_payload_dump,
353 .offload = nft_payload_offload,
356 static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum)
358 *sum = csum_fold(csum_add(csum_sub(~csum_unfold(*sum), fsum), tsum));
360 *sum = CSUM_MANGLED_0;
363 static bool nft_payload_udp_checksum(struct sk_buff *skb, unsigned int thoff)
365 struct udphdr *uh, _uh;
367 uh = skb_header_pointer(skb, thoff, sizeof(_uh), &_uh);
371 return (__force bool)uh->check;
374 static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt,
376 unsigned int *l4csum_offset)
378 switch (pkt->tprot) {
380 *l4csum_offset = offsetof(struct tcphdr, check);
383 if (!nft_payload_udp_checksum(skb, pkt->xt.thoff))
386 case IPPROTO_UDPLITE:
387 *l4csum_offset = offsetof(struct udphdr, check);
390 *l4csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
396 *l4csum_offset += pkt->xt.thoff;
400 static int nft_payload_l4csum_update(const struct nft_pktinfo *pkt,
402 __wsum fsum, __wsum tsum)
407 /* If we cannot determine layer 4 checksum offset or this packet doesn't
408 * require layer 4 checksum recalculation, skip this packet.
410 if (nft_payload_l4csum_offset(pkt, skb, &l4csum_offset) < 0)
413 if (skb_copy_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
416 /* Checksum mangling for an arbitrary amount of bytes, based on
417 * inet_proto_csum_replace*() functions.
419 if (skb->ip_summed != CHECKSUM_PARTIAL) {
420 nft_csum_replace(&sum, fsum, tsum);
421 if (skb->ip_summed == CHECKSUM_COMPLETE) {
422 skb->csum = ~csum_add(csum_sub(~(skb->csum), fsum),
426 sum = ~csum_fold(csum_add(csum_sub(csum_unfold(sum), fsum),
430 if (skb_ensure_writable(skb, l4csum_offset + sizeof(sum)) ||
431 skb_store_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
437 static int nft_payload_csum_inet(struct sk_buff *skb, const u32 *src,
438 __wsum fsum, __wsum tsum, int csum_offset)
442 if (skb_copy_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
445 nft_csum_replace(&sum, fsum, tsum);
446 if (skb_ensure_writable(skb, csum_offset + sizeof(sum)) ||
447 skb_store_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
453 static void nft_payload_set_eval(const struct nft_expr *expr,
454 struct nft_regs *regs,
455 const struct nft_pktinfo *pkt)
457 const struct nft_payload_set *priv = nft_expr_priv(expr);
458 struct sk_buff *skb = pkt->skb;
459 const u32 *src = ®s->data[priv->sreg];
460 int offset, csum_offset;
463 switch (priv->base) {
464 case NFT_PAYLOAD_LL_HEADER:
465 if (!skb_mac_header_was_set(skb))
467 offset = skb_mac_header(skb) - skb->data;
469 case NFT_PAYLOAD_NETWORK_HEADER:
470 offset = skb_network_offset(skb);
472 case NFT_PAYLOAD_TRANSPORT_HEADER:
475 offset = pkt->xt.thoff;
481 csum_offset = offset + priv->csum_offset;
482 offset += priv->offset;
484 if ((priv->csum_type == NFT_PAYLOAD_CSUM_INET || priv->csum_flags) &&
485 (priv->base != NFT_PAYLOAD_TRANSPORT_HEADER ||
486 skb->ip_summed != CHECKSUM_PARTIAL)) {
487 fsum = skb_checksum(skb, offset, priv->len, 0);
488 tsum = csum_partial(src, priv->len, 0);
490 if (priv->csum_type == NFT_PAYLOAD_CSUM_INET &&
491 nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset))
494 if (priv->csum_flags &&
495 nft_payload_l4csum_update(pkt, skb, fsum, tsum) < 0)
499 if (skb_ensure_writable(skb, max(offset + priv->len, 0)) ||
500 skb_store_bits(skb, offset, src, priv->len) < 0)
505 regs->verdict.code = NFT_BREAK;
508 static int nft_payload_set_init(const struct nft_ctx *ctx,
509 const struct nft_expr *expr,
510 const struct nlattr * const tb[])
512 struct nft_payload_set *priv = nft_expr_priv(expr);
514 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
515 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
516 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
517 priv->sreg = nft_parse_register(tb[NFTA_PAYLOAD_SREG]);
519 if (tb[NFTA_PAYLOAD_CSUM_TYPE])
521 ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE]));
522 if (tb[NFTA_PAYLOAD_CSUM_OFFSET])
524 ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_OFFSET]));
525 if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) {
528 flags = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_FLAGS]));
529 if (flags & ~NFT_PAYLOAD_L4CSUM_PSEUDOHDR)
532 priv->csum_flags = flags;
535 switch (priv->csum_type) {
536 case NFT_PAYLOAD_CSUM_NONE:
537 case NFT_PAYLOAD_CSUM_INET:
543 return nft_validate_register_load(priv->sreg, priv->len);
546 static int nft_payload_set_dump(struct sk_buff *skb, const struct nft_expr *expr)
548 const struct nft_payload_set *priv = nft_expr_priv(expr);
550 if (nft_dump_register(skb, NFTA_PAYLOAD_SREG, priv->sreg) ||
551 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
552 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
553 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)) ||
554 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_TYPE, htonl(priv->csum_type)) ||
555 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_OFFSET,
556 htonl(priv->csum_offset)) ||
557 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_FLAGS, htonl(priv->csum_flags)))
558 goto nla_put_failure;
565 static const struct nft_expr_ops nft_payload_set_ops = {
566 .type = &nft_payload_type,
567 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)),
568 .eval = nft_payload_set_eval,
569 .init = nft_payload_set_init,
570 .dump = nft_payload_set_dump,
573 static const struct nft_expr_ops *
574 nft_payload_select_ops(const struct nft_ctx *ctx,
575 const struct nlattr * const tb[])
577 enum nft_payload_bases base;
578 unsigned int offset, len;
580 if (tb[NFTA_PAYLOAD_BASE] == NULL ||
581 tb[NFTA_PAYLOAD_OFFSET] == NULL ||
582 tb[NFTA_PAYLOAD_LEN] == NULL)
583 return ERR_PTR(-EINVAL);
585 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
587 case NFT_PAYLOAD_LL_HEADER:
588 case NFT_PAYLOAD_NETWORK_HEADER:
589 case NFT_PAYLOAD_TRANSPORT_HEADER:
592 return ERR_PTR(-EOPNOTSUPP);
595 if (tb[NFTA_PAYLOAD_SREG] != NULL) {
596 if (tb[NFTA_PAYLOAD_DREG] != NULL)
597 return ERR_PTR(-EINVAL);
598 return &nft_payload_set_ops;
601 if (tb[NFTA_PAYLOAD_DREG] == NULL)
602 return ERR_PTR(-EINVAL);
604 offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
605 len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
607 if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) &&
608 base != NFT_PAYLOAD_LL_HEADER)
609 return &nft_payload_fast_ops;
611 return &nft_payload_ops;
614 struct nft_expr_type nft_payload_type __read_mostly = {
616 .select_ops = nft_payload_select_ops,
617 .policy = nft_payload_policy,
618 .maxattr = NFTA_PAYLOAD_MAX,
619 .owner = THIS_MODULE,